cafe-grader-judge-scripts

Commit r235:90afe1fa9431 public

parent | child
Description:
add feature to import_problem to automatically copy *.txt file and fix the judge script accordingly
Commit status:
[Not Reviewed]
References:
java
Diff options:
Raw Diff | Patch Diff | Download Diff | Show whitespace | Increase context
Comments:
0 Commit comments 0 Inline Comments
Unresolved TODOs:
There are no unresolved TODOs
Side by Side Unified
Expand All Files Collapse All Files Wide Mode Diff
Add another comment
Browse Files

r235:90afe1fa9431 - - 2 files changed: 12 inserted, 0 deleted

Show file before | Show file after | Raw diff | Download diff | Show whitespace Increase context Show commentsHide comments
@@ -52,185 +52,191
52 52 end
53 53
54 54 def print_usage
55 55 puts "using: import_problem_new name dir check [options]
56 56
57 57 where: name = problem_name (put '-' (dash) to use dir name)
58 58 dir = importing testcase directory
59 59 check = check script, which can be
60 60 'integer', 'text' (for standard script),
61 61 path_to_your_script, or
62 62 'wrapper:(path_to_your_wrapped_script)'
63 63 options: -t time-limit (in seconds)
64 64 -m memory-limit (in megabytes)
65 65
66 66 The script looks at test data files in the dir of the forms: *.in and
67 67 *.sol and import them to the evaluation dir for their environment,
68 68 based on their prefixes.
69 69
70 70 Currently supporting environments are:"
71 71
72 72 JUDGE_ENVIRONMENTS.each do |env|
73 73 prefix = ENV_INFO[env][:raw_prefix]
74 74 prefix = 'no prefix' if prefix==''
75 75 puts " * #{env}"
76 76 puts " import to: #{ENV_INFO[env][:ev_dir]}"
77 77 puts " prefix with: #{prefix} (e.g., #{ENV_INFO[env][:raw_prefix]}1.in, #{ENV_INFO[env][:raw_prefix]}5a.sol)"
78 78 end
79 79
80 80 puts"
81 81 For each environment, the script
82 82 * creates a directory for a problem in ev dir of that environment,
83 83 * copies testdata in the old format and create standard testcase config file
84 84 * copies a check script for grading
85 85 * creates a test_request template in the ev dir + '/test_request'
86 86
87 87 For wrapped checked script see comment in templates/check_wrapper for
88 88 information."
89 89 end
90 90
91 91 def count_testruns(testcase_dir, raw_prefix)
92 92 n = 0
93 93 begin
94 94 # check for test case n+1
95 95 if ((Dir["#{testcase_dir}/#{raw_prefix}#{n+1}.in"].length==0) and
96 96 (Dir["#{testcase_dir}/#{raw_prefix}#{n+1}[a-z].in"].length==0))
97 97 return n
98 98 end
99 99 n += 1
100 100 end while true
101 101 end
102 102
103 103 def create_dir_if_not_exists(dir)
104 104 if ! FileTest.exists? dir
105 105 FileUtils.mkdir(dir)
106 106 end
107 107 end
108 108
109 109 def import_problem(ev_dir, problem, testcase_dir, num_testruns, raw_prefix, check_script, options)
110 110 testrun_info = build_testrun_info_from_dir(num_testruns, testcase_dir, raw_prefix)
111 111
112 112 if !(FileTest.exists? ev_dir)
113 113 puts "Testdata dir (#{ev_dir}) not found."
114 114 return
115 115 end
116 116
117 117 problem_dir = "#{ev_dir}/#{problem}"
118 118
119 119 # start working
120 120 puts "creating directories"
121 121
122 122 create_dir_if_not_exists("#{problem_dir}")
123 123 create_dir_if_not_exists("#{problem_dir}/script")
124 124 create_dir_if_not_exists("#{problem_dir}/test_cases")
125 125
126 126 puts "copying testcases"
127 127
128 128 tr_num = 0
129 129
130 130 num_testcases = 0
131 131
132 132 testrun_info.each do |testrun|
133 133 tr_num += 1
134 134 puts "testrun: #{tr_num}"
135 135
136 136 testrun.each do |testcase_info|
137 137 testcase_num, testcase_fname = testcase_info
138 138
139 139 puts "copy #{testcase_fname} to #{testcase_num}"
140 140
141 141 create_dir_if_not_exists("#{problem_dir}/test_cases/#{testcase_num}")
142 142 copy_testcase("#{testcase_dir}",testcase_fname,"#{problem_dir}/test_cases/#{testcase_num}",testcase_num)
143 143
144 144 num_testcases += 1
145 145 end
146 146 end
147 147
148 + #also include any .txt files
149 + Dir.glob("#{testcase_dir}/*.txt") do |file|
150 + puts "copy data file #{file}"
151 + FileUtils.cp(file,"#{problem_dir}")
152 + end
153 +
148 154 # generating all_tests.cfg
149 155 puts "generating testcase config file"
150 156
151 157 template = File.open(SCRIPT_DIR + "/templates/all_tests.cfg.erb").read
152 158 all_test_cfg = ERB.new(template)
153 159
154 160 cfg_file = File.open("#{problem_dir}/test_cases/all_tests.cfg","w")
155 161 cfg_file.puts all_test_cfg.result binding
156 162 cfg_file.close
157 163
158 164 # copy check script
159 165 if res = /^wrapper:(.*)$/.match(check_script)
160 166 # wrapper script
161 167 check_script_fname = res[1]
162 168 script_name = File.basename(check_script_fname)
163 169 check_wrapper_template = File.open(SCRIPT_DIR + "/templates/check_wrapper").read
164 170 check_wrapper = ERB.new(check_wrapper_template)
165 171
166 172 check_file = File.open("#{problem_dir}/script/check","w")
167 173 check_file.puts check_wrapper.result binding
168 174 check_file.close
169 175
170 176 File.chmod(0755,"#{problem_dir}/script/check")
171 177
172 178 FileUtils.cp("#{check_script_fname}", "#{problem_dir}/script/#{script_name}")
173 179 else
174 180 if File.exists?(SCRIPT_DIR + "/templates/check.#{check_script}")
175 181 check_script_fname = SCRIPT_DIR + "/templates/check.#{check_script}"
176 182 else
177 183 check_script_fname = check_script
178 184 end
179 185 FileUtils.cp("#{check_script_fname}", "#{problem_dir}/script/check", :preserve => true)
180 186 end
181 187
182 188 # generating test_request directory
183 189 puts "generating test_request template"
184 190 FileUtils.mkdir_p("#{ev_dir}/test_request/#{problem}/script")
185 191 FileUtils.mkdir_p("#{ev_dir}/test_request/#{problem}/test_cases/1")
186 192
187 193 template = File.open(SCRIPT_DIR + "/templates/test_request_all_tests.cfg.erb").read
188 194 test_request_all_test_cfg = ERB.new(template)
189 195
190 196 cfg_file = File.open("#{ev_dir}/test_request/#{problem}/test_cases/all_tests.cfg","w")
191 197 cfg_file.puts test_request_all_test_cfg.result
192 198 cfg_file.close
193 199
194 200 FileUtils.cp("#{SCRIPT_DIR}/templates/check_empty",
195 201 "#{ev_dir}/test_request/#{problem}/script/check")
196 202 FileUtils.cp("#{SCRIPT_DIR}/templates/answer-1.txt",
197 203 "#{ev_dir}/test_request/#{problem}/test_cases/1")
198 204
199 205 puts "done"
200 206 end
201 207
202 208
203 209 SCRIPT_DIR = File.dirname(__FILE__)
204 210
205 211 # print usage
206 212 if (ARGV.length < 3) or (ARGV[2][0,1]=="-")
207 213 print_usage
208 214 exit(127)
209 215 end
210 216
211 217 # processing arguments
212 218 problem = ARGV[0]
213 219 testcase_dir = ARGV[1]
214 220 problem = File.basename(testcase_dir) if problem=="-"
215 221 check_script = ARGV[2]
216 222 options = {:time_limit => 1, :mem_limit => 16}
217 223 process_options(options)
218 224
219 225 JUDGE_ENVIRONMENTS.each do |env|
220 226 ev_dir = ENV_INFO[env][:ev_dir]
221 227 raw_prefix = ENV_INFO[env][:raw_prefix]
222 228
223 229 num_testruns = count_testruns(testcase_dir,raw_prefix)
224 230
225 231 puts ""
226 232 puts "*** Environment: #{env} (#{num_testruns} test runs) ***"
227 233 puts ""
228 234
229 235 import_problem(ev_dir,
230 236 problem,
231 237 testcase_dir,
232 238 num_testruns,
233 239 raw_prefix,
234 240 check_script,
235 241 options)
236 242 end
Show file before | Show file after | Raw diff | Download diff | Show whitespace Increase context Show commentsHide comments
@@ -47,134 +47,140
47 47 if ARGV.length < 2 || ARGV.length > 4
48 48 puts "Usage: judge <language> <program-source> [<test-result-directory>] [<sandbox-directory>]"
49 49 puts " <sandbox-directory> is defaulted to ./sandbox"
50 50 puts " <test-result-directory> is defaulted to ./test-result"
51 51 puts "WARNING: The judge script will forcefully create the (implicitly and explicitly) specified directories and remove anything inside it."
52 52 exit(127)
53 53 end
54 54
55 55 language = ARGV[0]
56 56 if language != "c" && language != "c++" && language != "pas" && language != "java" && language != "ruby" && language != "python" && language != "php" && language != "haskell"
57 57 log "You specified a language that is not supported: #{language}."
58 58 exit(127)
59 59 end
60 60
61 61 source_file = ARGV[1]
62 62 ENV['SOURCE_NAME'] = source_file
63 63 if File.exist?(source_file) == false
64 64 log "The source file does not exist."
65 65 exit(127)
66 66 end
67 67
68 68 log "Making test result and sandbox directories..."
69 69
70 70 current_dir = FileUtils.pwd
71 71 current_dir.strip!
72 72
73 73 if ARGV.length >= 3
74 74 test_result_dir = ARGV[2]
75 75 else
76 76 test_result_dir = "#{current_dir}/test-result"
77 77 end
78 78
79 79 log "Test result directory: #{test_result_dir}"
80 80 clear_and_create_empty_dir(test_result_dir)
81 81
82 82 if ARGV.length >= 4
83 83 sandbox_dir = ARGV[3]
84 84 else
85 85 sandbox_dir = "#{current_dir}/sandbox"
86 86 end
87 87 log "Sandbox directory: #{sandbox_dir}"
88 88 clear_and_create_empty_dir(sandbox_dir)
89 89
90 90 # Compile
91 91 log
92 92 log "Compiling..."
93 93 call_and_log("Cannot copy the source file to #{sandbox_dir}") {
94 94 FileUtils.cp(source_file, sandbox_dir)
95 95 }
96 96 begin
97 97 Dir.chdir sandbox_dir
98 98 rescue
99 99 log "ERROR: Cannot change directory to #{sandbox_dir}."
100 100 exit(127)
101 101 end
102 102 execute("#{problem_home}/script/compile #{language} #{source_file}", "Compilation error!")
103 103 compile_message = open("compiler_message").read
104 104 compile_message.strip!
105 105 call_and_log("Cannot move the compiler message to #{test_result_dir}.") {
106 106 FileUtils.mv("compiler_message", test_result_dir)
107 107 }
108 108 if !FileTest.exist?("a.out")
109 109 log "Cannot compile the source code. See message in #{test_result_dir}/compile_message"
110 110 exit(127)
111 111 else
112 112 call_and_log("Cannot move the compiled program to #{test_result_dir}") {
113 113 FileUtils.mv("a.out",test_result_dir)
114 114 if language == "java" then Dir["*.class"].each { |file| FileUtils.mv(file,test_result_dir)} end
115 115 if language == "python" then Dir["*.pyc"].each { |file| FileUtils.mv(file,test_result_dir)} end
116 116 }
117 117 FileUtils.rm_rf("#{sandbox_dir}/.")
118 118 end
119 119
120 120 require "#{problem_home}/script/test_dsl.rb"
121 121 load "#{problem_home}/test_cases/all_tests.cfg"
122 122 problem = Problem.get_instance
123 123
124 124 if problem.well_formed? == false
125 125 log "The problem specification is not well formed."
126 126 exit(127)
127 127 end
128 128
129 129 # Doing the testing.
130 130 (1..(problem.num_tests)).each do |test_num|
131 131
132 132 $stdout.print "[#{test_num}]"
133 133 $stdout.flush
134 134
135 135 log "Test number: #{test_num}"
136 136
137 137 call_and_log("Cannot copy the compiled program into #{sandbox_dir}") {
138 138 FileUtils.cp("#{test_result_dir}/a.out", sandbox_dir, :preserve => true)
139 139 if language == "java" then Dir["#{test_result_dir}/*.class"].each { |file| FileUtils.cp(file,sandbox_dir)} end
140 140 if language == "python" then Dir["#{test_result_dir}/*.pyc"].each { |file| FileUtils.cp(file,sandbox_dir)} end
141 141 }
142 142
143 + #additionally copy any extra .txt file
144 + data_files = Dir[problem_home + '/*.txt']
145 + data_files.each do |file|
146 + FileUtils.cp(file,sandbox_dir)
147 + end
148 +
143 149 begin
144 150 execute("#{problem_home}/script/run #{language} #{test_num} ", "Error occured during execution of the run script")
145 151 rescue
146 152 # do nothing
147 153 end
148 154
149 155 call_and_log("Cannot create directory #{test_result_dir}/#{test_num}") {
150 156 FileUtils.mkdir "#{test_result_dir}/#{test_num}"
151 157 }
152 158 call_and_log("Cannot copy the result file into #{test_result_dir}/#{test_num}") {
153 159 FileUtils.mv "#{sandbox_dir}/result", "#{test_result_dir}/#{test_num}"
154 160 }
155 161 call_and_log("Cannot copy the comment file into #{test_result_dir}/#{test_num}") {
156 162 FileUtils.mv "#{sandbox_dir}/comment", "#{test_result_dir}/#{test_num}"
157 163 }
158 164 call_and_log("Cannot copy the output file into #{test_result_dir}/#{test_num}") {
159 165 FileUtils.mv "#{sandbox_dir}/output.txt", "#{test_result_dir}/#{test_num}"
160 166 }
161 167 call_and_log("Cannot clear #{sandbox_dir}") {
162 168 FileUtils.rm_rf(Dir.glob("#{sandbox_dir}/*"), :secure => true)
163 169 }
164 170 end
165 171
166 172 $stdout.print "[done]\n"
167 173
168 174 # Grade
169 175 log
170 176 log "Grading..."
171 177 begin
172 178 Dir.chdir test_result_dir
173 179 rescue
174 180 log "ERROR: Cannot change directory to #{test_result_dir}."
175 181 exit(127)
176 182 end
177 183 execute("#{problem_home}/script/grade", "An error occured during grading!")
178 184
179 185 log
180 186 log "All done!"
You need to be logged in to leave comments. Login now