cafe-grader-judge-scripts

Commit r66:c03ebf30fb10 public

parent | child
Description:
fixed test run scoring bug: now it takes the minimum of each test case's score git-svn-id: http://theory.cpe.ku.ac.th/grader/judge/trunk/scripts@273 6386c4cd-e34a-4fa8-8920-d93eb39b512e
Commit status:
[Not Reviewed]
References:
default
Diff options:
Raw Diff | Patch Diff | Download Diff | Ignore whitespace | Increase context
Comments:
0 Commit comments 0 Inline Comments
Unresolved TODOs:
There are no unresolved TODOs
Side by Side Unified
Expand All Files Collapse All Files Wide Mode Diff
Add another comment
Browse Files

r66:c03ebf30fb10 - - 4 files changed: 19 inserted, 18 deleted

Show file before | Show file after | Raw diff | Download diff | Ignore whitespace Increase context Show commentsHide comments
@@ -1,240 +1,242
1 1 #
2 2 # This part contains various test_request helpers for interfacing
3 3 # with Grader::Engine. There are TestRequestRoomMaker and
4 4 # TestRequestReporter.
5 5
6 6 module Grader
7 7
8 8 #
9 9 # A TestRequestRoomMaker is a helper object for Engine
10 10 # - finds grading room: in user_result_dir/(user)/test_request/ ...
11 11 # - prepare problem configuration for grading --- basically it copy
12 12 # all config files, and copy user's input into the testcase
13 13 # directory. First, it finds the template from problem template
14 14 # directory; if it can't find a template, it'll use the template
15 15 # from default template.
16 16 class TestRequestRoomMaker
17 17 def initialize
18 18 @config = Grader::Configuration.get_instance
19 19 end
20 20
21 21 def produce_grading_room(test_request)
22 22 grading_room = grading_room_dir(test_request)
23 23 FileUtils.mkdir_p(grading_room)
24 24
25 25 #
26 26 # Also copy additional submitted file to this directory as well.
27 27 # The program would see this file only if it is copied
28 28 # to the sandbox directory later. The run script should do it.
29 29 #
30 - cmd = "cp #{test_request.input_file_name}.files/* #{grading_room}"
31 - system(cmd)
30 + if FileTest.exists?("#{test_request.input_file_name}.files")
31 + cmd = "cp #{test_request.input_file_name}.files/* #{grading_room}"
32 + system(cmd)
33 + end
32 34
33 35 grading_room
34 36 end
35 37
36 38 def find_problem_home(test_request)
37 39 problem_name = test_request.problem_name
38 40
39 41 template_dir = "#{@config.test_request_problem_templates_dir}/" + problem_name
40 42
41 43 raise "Test Request: error template not found" if !File.exists?(template_dir)
42 44
43 45 problem_home = problem_home_dir(test_request)
44 46 FileUtils.mkdir_p(problem_home)
45 47
46 48 copy_problem_template(template_dir,problem_home)
47 49 link_input_file(test_request,problem_home)
48 50
49 51 problem_home
50 52 end
51 53
52 54 def save_source(test_request,source_name)
53 55 dir = self.produce_grading_room(test_request)
54 56 submission = test_request.submission
55 57 f = File.open("#{dir}/#{source_name}","w")
56 58 f.write(submission.source)
57 59 f.close
58 60 end
59 61
60 62 def clean_up(test_request)
61 63 problem_home = problem_home_dir(test_request)
62 64 remove_data_files(problem_home)
63 65 end
64 66
65 67 protected
66 68 def grading_room_dir(test_request)
67 69 problem_name = test_request.problem_name
68 70 user = test_request.user
69 71 grading_room = "#{@config.user_result_dir}" +
70 72 "/#{user.login}/test_request" +
71 73 "/#{problem_name}/#{test_request.id}"
72 74 grading_room
73 75 end
74 76
75 77 def problem_home_dir(test_request)
76 78 problem_name = test_request.problem_name
77 79 user = test_request.user
78 80 "#{@config.user_result_dir}" +
79 81 "/#{user.login}/test_request/#{problem_name}"
80 82 end
81 83
82 84 def copy_problem_template(template_dir,problem_home)
83 85 cmd = "cp -R #{template_dir}/* #{problem_home}"
84 86 system_and_raise_when_fail(cmd,"Test Request: cannot copy problem template")
85 87 end
86 88
87 89 def link_input_file(test_request,problem_home)
88 90 input_fname = "#{test_request.input_file_name}"
89 91 if !File.exists?(input_fname)
90 92 raise "Test Request: input file not found."
91 93 end
92 94
93 95 input_fname_problem_home = "#{problem_home}/test_cases/1/input-1.txt"
94 96 if File.exists?(input_fname_problem_home)
95 97 FileUtils.rm([input_fname_problem_home], :force => true)
96 98 end
97 99
98 100 cmd = "ln -s #{input_fname} #{input_fname_problem_home}"
99 101 system_and_raise_when_fail(cmd,"Test Request: cannot link input file")
100 102 end
101 103
102 104 def remove_data_files(problem_home)
103 105 if File.exists?("#{problem_home}/test_cases/1/input-1.txt")
104 106 cmd = "rm #{problem_home}/test_cases/1/*"
105 107 system_and_raise_when_fail(cmd,"Test Request: cannot remove data files")
106 108 end
107 109 end
108 110
109 111 def system_and_raise_when_fail(cmd,msg)
110 112 if !system(cmd)
111 113 raise msg
112 114 end
113 115 end
114 116
115 117 end
116 118
117 119 class TestRequestReporter
118 120 def initialize
119 121 @config = Grader::Configuration.get_instance
120 122 end
121 123
122 124 def report(test_request,test_result_dir)
123 125 save_result(test_request,read_result(test_result_dir))
124 126 end
125 127
126 128 def report_error(test_request, msg)
127 129 save_result(test_request, {:running_stat => {
128 130 :msg => "#{msg}",
129 131 :running_time => nil,
130 132 :exit_status => "Some error occured. Program did not run",
131 133 :memory_usage => nil
132 134 }})
133 135 end
134 136
135 137 protected
136 138 def read_result(test_result_dir)
137 139 # TODO:
138 140 cmp_msg_fname = "#{test_result_dir}/compiler_message"
139 141 cmp_file = File.open(cmp_msg_fname)
140 142 cmp_msg = cmp_file.read
141 143 cmp_file.close
142 144
143 145 result_file_name = "#{test_result_dir}/1/result"
144 146
145 147 if File.exists?(result_file_name)
146 148 output_file_name = "#{test_result_dir}/1/output.txt"
147 149 results = File.open("#{test_result_dir}/1/result").readlines
148 150 stat = extract_running_stat(results)
149 151
150 152 return {
151 153 :output_file_name => output_file_name,
152 154 :running_stat => stat,
153 155 :comment => "",
154 156 :cmp_msg => cmp_msg}
155 157 else
156 158 return {
157 159 :running_stat => nil,
158 160 :comment => "Compilation error",
159 161 :cmp_msg => cmp_msg}
160 162 end
161 163 end
162 164
163 165 def extract_running_stat(results)
164 166 running_stat_line = results[-1]
165 167
166 168 # extract exit status line
167 169 run_stat = ""
168 170 if !(/[Cc]orrect/.match(results[0]))
169 171 run_stat = results[0].chomp
170 172 else
171 173 run_stat = 'Program exited normally'
172 174 end
173 175
174 176 # extract running time
175 177 if res = /r(.*)u(.*)s/.match(running_stat_line)
176 178 seconds = (res[1].to_f + res[2].to_f)
177 179 time_stat = "Time used: #{seconds} sec."
178 180 else
179 181 seconds = nil
180 182 time_stat = "Time used: n/a sec."
181 183 end
182 184
183 185 # extract memory usage
184 186 if res = /s(.*)m/.match(running_stat_line)
185 187 memory_used = res[1].to_i
186 188 else
187 189 memory_used = -1
188 190 end
189 191
190 192 return {
191 193 :msg => "#{run_stat}\n#{time_stat}",
192 194 :running_time => seconds,
193 195 :exit_status => run_stat,
194 196 :memory_usage => memory_used
195 197 }
196 198 end
197 199
198 200 def save_result(test_request,result)
199 201 if result[:output_file_name]!=nil
200 202 test_request.output_file_name = link_output_file(test_request,
201 203 result[:output_file_name])
202 204 end
203 205 test_request.graded_at = Time.now
204 206 test_request.compiler_message = (result[:cmp_msg] or '')
205 207 test_request.grader_comment = (result[:comment] or '')
206 208 if result[:running_stat]!=nil
207 209 test_request.running_stat = (result[:running_stat][:msg] or '')
208 210 test_request.running_time = (result[:running_stat][:running_time] or nil)
209 211 test_request.exit_status = result[:running_stat][:exit_status]
210 212 test_request.memory_usage = result[:running_stat][:memory_usage]
211 213 else
212 214 test_request.running_stat = ''
213 215 end
214 216 test_request.save
215 217 end
216 218
217 219 protected
218 220 def link_output_file(test_request, fname)
219 221 target_file_name = random_output_file_name(test_request.user,
220 222 test_request.problem)
221 223 FileUtils.mkdir_p(File.dirname(target_file_name))
222 224 cmd = "ln -s #{fname} #{target_file_name}"
223 225 if !system(cmd)
224 226 raise "TestRequestReporter: cannot move output file"
225 227 end
226 228 return target_file_name
227 229 end
228 230
229 231 def random_output_file_name(user,problem)
230 232 problem_name = TestRequest.name_of(problem)
231 233 begin
232 234 tmpname = "#{@config.test_request_output_base_dir}" +
233 235 "/#{user.login}/#{problem_name}/#{rand(10000)}"
234 236 end while File.exists?(tmpname)
235 237 tmpname
236 238 end
237 239
238 240 end
239 241
240 242 end
Show file before | Show file after | Raw diff | Download diff | Ignore whitespace Increase context Show commentsHide comments
@@ -1,108 +1,106
1 1 #!/usr/bin/ruby
2 2
3 3 CORRECT_MARK = 'P'
4 4 INCORRECT_MARK = '-'
5 5 TIMEOUT_MARK = 'T'
6 6 RUN_ERROR_MARK = 'x'
7 7
8 8 def log(str='')
9 9 if ENV['TALKATIVE']!=nil
10 10 puts str
11 11 end
12 12 if ENV['GRADER_LOGGING']!=nil
13 13 log_fname = ENV['GRADER_LOGGING']
14 14 fp = File.open(log_fname,"a")
15 15 fp.puts("grade: #{Time.new.strftime("%H:%M")} #{str}")
16 16 fp.close
17 17 end
18 18 end
19 19
20 20 def char_comment(comment)
21 21 if comment =~ /[Ii]ncorrect/
22 22 INCORRECT_MARK
23 23 elsif comment =~ /[Cc]orrect/
24 24 CORRECT_MARK
25 25 elsif comment =~ /[Tt]ime/
26 26 TIMEOUT_MARK
27 27 elsif res = /^[Cc]omment:(.*)$/.match(comment)
28 28 res[1]
29 29 else
30 30 RUN_ERROR_MARK # these are run time errors
31 31 end
32 32 end
33 33
34 34 problem_home = ENV['PROBLEM_HOME']
35 35 require "#{problem_home}/script/test_dsl.rb"
36 36 load "#{problem_home}/test_cases/all_tests.cfg"
37 37 problem = Problem.get_instance
38 38
39 39 if problem.well_formed? == false
40 40 log "The problem specification is not well formed."
41 41 exit(127)
42 42 end
43 43
44 44 all_score = 0
45 45 all_comment = ''
46 46 (1..(problem.runs.length-1)).each do |k|
47 47 log "grade run #{k}"
48 48 run = problem.runs[k]
49 - run_score = 0
49 + run_score = nil
50 50 run_comment = ''
51 51 run_comment_short = ''
52 52 run.tests.each do |test_num|
53 53 result_file_name = "#{test_num}/result"
54 54 if not File.exists?(result_file_name)
55 55 run_comment += "result file for test #{test_num} not found\n"
56 56 run_comment_short += RUN_ERROR_MARK
57 57 log "Cannot find the file #{test_num}/result!"
58 58 else
59 59 result_file = File.new(result_file_name, "r")
60 60 result_file_lines = result_file.readlines
61 61 if result_file_lines.length>=2
62 - run_score = run_score + result_file_lines[1].to_i
62 + current_run_score = result_file_lines[1].to_i
63 63 run_comment += result_file_lines[0]
64 64 run_comment_short += char_comment(result_file_lines[0].chomp)
65 65 else
66 + current_run_score = 0
66 67 run_comment += "result file for test #{test_num} error\n"
67 68 run_comment_short += RUN_ERROR_MARK
68 69 log "Error in #{test_num}/result!"
69 70 end
71 +
72 + # the score of this run should be the minimum of the score for
73 + # each test case
74 + if (run_score==nil) or (run_score>current_run_score)
75 + run_score = current_run_score
76 + end
70 77 result_file.close
71 78 end
72 79 end
73 80
74 - # find total score for this run
75 - run_total_score = 0
76 - problem = Problem.get_instance
77 - run.tests.each { |test_num| run_total_score += problem.get_score(test_num) }
78 -
79 - if run_total_score!=run_score # fail in some test cases, fail the run
80 - run_score = 0
81 - end
82 -
83 81 run_result_file = File.new("result-#{k}", "w")
84 82 run_result_file.write run_score
85 83 run_result_file.write "\n"
86 84 run_result_file.close
87 85
88 86 run_comment_file = File.new("comment-#{k}", "w")
89 87 run_comment_file.write "#{run_comment}\n"
90 88 run_comment_file.close
91 89
92 90 all_score = all_score + run_score
93 91
94 92 # append comment for test run with many test cases
95 93 if run.tests.length > 1
96 94 run_comment_short = '[' + run_comment_short + ']'
97 95 end
98 96 all_comment += run_comment_short
99 97 end
100 98
101 99 result_file = File.new("result", "w")
102 100 result_file.write all_score
103 101 result_file.write "\n"
104 102 result_file.close
105 103
106 104 comment_file = File.new("comment", "w")
107 105 comment_file.write "#{all_comment}\n"
108 106 comment_file.close
Show file before | Show file after | Raw diff | Download diff | Ignore whitespace Increase context Show commentsHide comments
@@ -1,20 +1,20
1 1 problem do
2 2 num_tests <%= num_testcases %>
3 3 full_score <%= num_testruns*10 %>
4 4 time_limit_each <%= options[:time_limit] %>
5 5 mem_limit_each <%= options[:mem_limit] %>
6 6 score_each 10
7 7
8 8 <% tr_num = 0 %>
9 9 <% testrun_info.each do |testrun| %>
10 10 <% tr_num += 1 %>
11 11 run <%= tr_num %> do
12 12 tests <%= (testrun.collect {|testcase| testcase[0]}).join(", ") %>
13 13 <% if testrun.length==1 %>
14 14 scores 10
15 15 <% else %>
16 - scores 10 <% (testrun.length-1).times do %>,0 <% end %>
16 + scores 10 <% (testrun.length-1).times do %>,10 <% end %>
17 17 <% end %>
18 18 end
19 19 <% end %>
20 20 end
Show file before | Show file after | Raw diff | Download diff | Ignore whitespace Increase context Show commentsHide comments
@@ -1,39 +1,40
1 1 problem do
2 2 num_tests 10
3 3 full_score 135
4 4 time_limit_each 1
5 5 mem_limit_each 11
6 6 score_each 10
7 7
8 8 run 1 do
9 9 tests 1, 2
10 - scores 10, 20
10 + scores 30, 30
11 11 time_limits 1, 2
12 12 mem_limits 5, 6
13 13 end
14 14
15 15 run 2 do
16 16 tests 3, 4, 5, 6, 7
17 - score_each 10
17 + score_each 50
18 18 time_limit_each 3
19 19 mem_limit_each 3
20 20 end
21 21
22 22 run 3 do
23 23 tests 8, 9, 10
24 24 end
25 25
26 26 test 8 do
27 - score 30
27 + score 55
28 28 time_limit 3
29 29 mem_limit 10
30 30 end
31 31
32 32 test 9 do
33 - score 15
33 + score 55
34 34 end
35 35
36 36 test 10 do
37 + score 55
37 38 time_limit 1
38 39 end
39 40 end
You need to be logged in to leave comments. Login now