cafe-grader-judge-scripts

Commit r206:cf0534046f00 public

parent | child
Description:
Merge pull request #10 from wytesk133/master Fixed some Ruby scripts
Commit status:
[Not Reviewed]
References:
merge default
Diff options:
Raw Diff | Patch Diff | Download Diff | Ignore whitespace | Increase context
Comments:
0 Commit comments 0 Inline Comments
Unresolved TODOs:
There are no unresolved TODOs
Side by Side Unified
Expand All Files Collapse All Files Wide Mode Diff
Add another comment
Browse Files

r206:cf0534046f00 - - 8 files changed: 11 inserted, 11 deleted

Show file before | Show file after | Raw diff | Download diff | Ignore whitespace Increase context Show commentsHide comments
@@ -1,73 +1,73
1 - #!/usr/bin/ruby
1 + #!/usr/bin/env ruby
2 2
3 3 # new_problem:
4 4 # * creates a directory for a problem in the current directory,
5 5 # * create standard testcase config file
6 6
7 7 require 'erb'
8 8
9 9 def process_options(options)
10 10 i = 2
11 11 while i<ARGV.length
12 12 if ARGV[i]=='-t'
13 13 options[:time_limit] = ARGV[i+1].to_i if ARGV.length>i+1
14 14 i += 1
15 15 end
16 16 if ARGV[i]=='-m'
17 17 options[:mem_limit] = ARGV[i+1].to_i if ARGV.length>i+1
18 18 i += 1
19 19 end
20 20 i += 1
21 21 end
22 22 end
23 23
24 24
25 25 puts "This script is out of dated, shall be fixed soon"
26 26 puts "Right now, you can create raw_ev and import"
27 27 exit(0)
28 28
29 29 GRADER_DIR = File.dirname(__FILE__)
30 30
31 31 # print usage
32 32 if ARGV.length < 2
33 33 puts <<USAGE
34 34 using: new_problem problem number_of_testcase [options]
35 35 * creates a directory for a problem in the current directory,
36 36 * create standard testcase config file
37 37 * options: -t time-limit (in seconds)
38 38 -m mem-limit (in MB)
39 39 USAGE
40 40 exit(127)
41 41 end
42 42
43 43 # processing arguments
44 44 problem = ARGV[0]
45 45 num_testcases = ARGV[1].to_i
46 46
47 47 options = {:time_limit => 1, :mem_limit => 16}
48 48 process_options(options)
49 49
50 50 # start working
51 51 puts "creating directories"
52 52
53 53 system("mkdir #{problem}")
54 54 system("mkdir #{problem}/script")
55 55 system("mkdir #{problem}/test_cases")
56 56
57 57 puts "creating testcases directories"
58 58
59 59 1.upto(num_testcases) do |i|
60 60 system("mkdir #{problem}/test_cases/#{i}")
61 61 end
62 62
63 63 # generating all_tests.cfg
64 64 puts "generating testcase config file"
65 65
66 66 template = File.open(File.dirname(__FILE__) + "/templates/all_tests.cfg.erb").read
67 67 all_test_cfg = ERB.new(template)
68 68
69 69 cfg_file = File.open("#{problem}/test_cases/all_tests.cfg","w")
70 70 cfg_file.puts all_test_cfg.result
71 71 cfg_file.close
72 72
73 73 puts "done"
Show file before | Show file after | Raw diff | Download diff | Ignore whitespace Increase context Show commentsHide comments
@@ -1,48 +1,48
1 - #!/usr/bin/ruby
1 + #!/usr/bin/env ruby
2 2
3 3 #
4 4 # This is a check script wrapper. It read all required information
5 5 # and call a real check script call REAL_CHECK_SCRIPT in directory
6 6 # [problem_home]/script
7 7 #
8 8
9 9 REAL_CHECK_SCRIPT = "<%= script_name %>"
10 10
11 11 # The REAL_CHECK_SCRIPT is called with:
12 12 #
13 13 # (script) <lang> <test-num> <in-file> <out-file> <ans-file> <full-score>
14 14 #
15 15 # and REAL_CHECK_SCRIPT's output to standard out is redirected to
16 16 # 'check_result' as required by normal check script.
17 17
18 18 problem_home = ENV['PROBLEM_HOME']
19 19 require "#{problem_home}/script/test_dsl.rb"
20 20
21 21 if ARGV.length < 2
22 22 puts "Usage: check <language> <test-number> [<output-file>]"
23 23 exit(0)
24 24 end
25 25
26 26 language = ARGV[0]
27 27 test_num = ARGV[1].to_i
28 28 if ARGV.length >= 3
29 29 output_file_name = ARGV[2]
30 30 else
31 31 output_file_name = "output.txt"
32 32 end
33 33
34 34 load "#{problem_home}/test_cases/all_tests.cfg"
35 35 problem = Problem.get_instance
36 36
37 37 answer_file_name = "#{problem_home}/test_cases/#{test_num}/answer-#{test_num}.txt"
38 38 input_file_name = "#{problem_home}/test_cases/#{test_num}/input-#{test_num}.txt"
39 39
40 40 score = problem.get_score(test_num)
41 41
42 42 cmd = "#{problem_home}/script/#{REAL_CHECK_SCRIPT} " +
43 43 "#{language} #{test_num} #{input_file_name} #{output_file_name} " +
44 44 "#{answer_file_name} #{score} > check_result"
45 45
46 46 #puts "wrapper-CMD: #{cmd}"
47 47
48 48 system(cmd)
Show file before | Show file after | Raw diff | Download diff | Ignore whitespace Increase context Show commentsHide comments
@@ -1,59 +1,59
1 - #!/usr/bin/ruby
1 + #!/usr/bin/env ruby
2 2
3 3 problem_home = ENV['PROBLEM_HOME']
4 4 require "#{problem_home}/script/test_dsl.rb"
5 5
6 6 if ARGV.length < 2
7 7 puts "Usage: check <language> <test-number> [<output-file>]"
8 8 exit(0)
9 9 end
10 10
11 11 language = ARGV[0]
12 12 test_num = ARGV[1].to_i
13 13 if ARGV.length >= 3
14 14 output_file_name = ARGV[2]
15 15 else
16 16 output_file_name = "output.txt"
17 17 end
18 18
19 19 load "#{problem_home}/test_cases/all_tests.cfg"
20 20 problem = Problem.get_instance
21 21
22 22 output_file = File.new(output_file_name, "r")
23 23 answer_file = File.new("#{problem_home}/test_cases/#{test_num}/answer-#{test_num}.txt")
24 24 result_file = File.new("check_result", "w")
25 25
26 26 output_file_content = output_file.read
27 27 answer_file_content = answer_file.read
28 28
29 29 report_correct = lambda {
30 30 result_file.write "Correct\n"
31 31 result_file.write problem.get_score(test_num)
32 32 result_file.write "\n"
33 33 result_file.close
34 34 exit(0)
35 35 }
36 36
37 37 report_wrong = lambda {
38 38 result_file.write "Incorrect\n"
39 39 result_file.write "0\n"
40 40 result_file.close
41 41 exit(0)
42 42 }
43 43
44 44 ##################
45 45 # Your code here #
46 46 ##################
47 47 num_pattern = /^[0-9]*/
48 48 if (output_file_content =~ num_pattern) == nil
49 49 report_wrong.call
50 50 end
51 51
52 52 output_i = output_file_content.to_i
53 53 answer_i = answer_file_content.to_i
54 54
55 55 if output_i == answer_i
56 56 report_correct.call
57 57 else
58 58 report_wrong.call
59 59 end
Show file before | Show file after | Raw diff | Download diff | Ignore whitespace Increase context Show commentsHide comments
@@ -1,59 +1,59
1 - #!/usr/bin/ruby
1 + #!/usr/bin/env ruby
2 2
3 3 problem_home = ENV['PROBLEM_HOME']
4 4 require "#{problem_home}/script/test_dsl.rb"
5 5
6 6 if ARGV.length < 2
7 7 puts "Usage: check <language> <test-number> [<output-file>]"
8 8 exit(0)
9 9 end
10 10
11 11 language = ARGV[0]
12 12 test_num = ARGV[1].to_i
13 13 if ARGV.length >= 3
14 14 output_file_name = ARGV[2]
15 15 else
16 16 output_file_name = "output.txt"
17 17 end
18 18
19 19 load "#{problem_home}/test_cases/all_tests.cfg"
20 20 problem = Problem.get_instance
21 21
22 22 output_file = File.new(output_file_name, "r")
23 23 answer_file = File.new("#{problem_home}/test_cases/#{test_num}/answer-#{test_num}.txt")
24 24 result_file = File.new("check_result", "w")
25 25
26 26 output_file_content = output_file.read
27 27 answer_file_content = answer_file.read
28 28
29 29 report_correct = lambda {
30 30 result_file.write "Correct\n"
31 31 result_file.write problem.get_score(test_num)
32 32 result_file.write "\n"
33 33 result_file.close
34 34 exit(0)
35 35 }
36 36
37 37 report_wrong = lambda {
38 38 result_file.write "Incorrect\n"
39 39 result_file.write "0\n"
40 40 result_file.close
41 41 exit(0)
42 42 }
43 43
44 44 ##################
45 45 # Your code here #
46 46 ##################
47 47 num_pattern = /^[0-9]*/
48 48 if (output_file_content =~ num_pattern) == nil
49 49 report_wrong.call
50 50 end
51 51
52 52 output_i = output_file_content.to_i
53 53 answer_i = answer_file_content.to_i
54 54
55 55 if output_i == answer_i
56 56 report_correct.call
57 57 else
58 58 report_wrong.call
59 59 end
Show file before | Show file after | Raw diff | Download diff | Ignore whitespace Increase context Show commentsHide comments
@@ -1,59 +1,59
1 - #!/usr/bin/ruby
1 + #!/usr/bin/env ruby
2 2
3 3 problem_home = ENV['PROBLEM_HOME']
4 4 require "#{problem_home}/script/test_dsl.rb"
5 5
6 6 if ARGV.length < 2
7 7 puts "Usage: check <language> <test-number> [<output-file>]"
8 8 exit(0)
9 9 end
10 10
11 11 language = ARGV[0]
12 12 test_num = ARGV[1].to_i
13 13 if ARGV.length >= 3
14 14 output_file_name = ARGV[2]
15 15 else
16 16 output_file_name = "output.txt"
17 17 end
18 18
19 19 load "#{problem_home}/test_cases/all_tests.cfg"
20 20 problem = Problem.get_instance
21 21
22 22 output_file = File.new(output_file_name, "r")
23 23 answer_file = File.new("#{problem_home}/test_cases/#{test_num}/answer-#{test_num}.txt")
24 24 result_file = File.new("check_result", "w")
25 25
26 26 output_file_content = output_file.read
27 27 answer_file_content = answer_file.read
28 28
29 29 report_correct = lambda {
30 30 result_file.write "Correct\n"
31 31 result_file.write problem.get_score(test_num)
32 32 result_file.write "\n"
33 33 result_file.close
34 34 exit(0)
35 35 }
36 36
37 37 report_wrong = lambda {
38 38 result_file.write "Incorrect\n"
39 39 result_file.write "0\n"
40 40 result_file.close
41 41 exit(0)
42 42 }
43 43
44 44 ##################
45 45 # Your code here #
46 46 ##################
47 47 num_pattern = /^[0-9]*/
48 48 if (output_file_content =~ num_pattern) == nil
49 49 report_wrong.call
50 50 end
51 51
52 52 output_i = output_file_content.to_i
53 53 answer_i = answer_file_content.to_i
54 54
55 55 if output_i == answer_i
56 56 report_correct.call
57 57 else
58 58 report_wrong.call
59 59 end
Show file before | Show file after | Raw diff | Download diff | Ignore whitespace Increase context Show commentsHide comments
@@ -1,64 +1,64
1 - #!/usr/bin/ruby
1 + #!/usr/bin/env ruby
2 2
3 3 problem_home = ENV['PROBLEM_HOME']
4 4 require "#{problem_home}/script/test_dsl.rb"
5 5
6 6 if ARGV.length < 2
7 7 puts "Usage: check <language> <test-number> [<output-file>]"
8 8 exit(0)
9 9 end
10 10
11 11 language = ARGV[0]
12 12 test_num = ARGV[1].to_i
13 13 if ARGV.length >= 3
14 14 output_file_name = ARGV[2]
15 15 else
16 16 output_file_name = "output.txt"
17 17 end
18 18
19 19 load "#{problem_home}/test_cases/all_tests.cfg"
20 20 problem = Problem.get_instance
21 21
22 22 output_file = File.new(output_file_name, "r")
23 23 answer_file = File.new("#{problem_home}/test_cases/#{test_num}/answer-#{test_num}.txt")
24 24 result_file = File.new("check_result", "w")
25 25
26 26 output_file_content = output_file.read
27 27 answer_file_content = answer_file.read
28 28
29 29 report_correct = lambda {
30 30 result_file.write "Correct\n"
31 31 result_file.write problem.get_score(test_num)
32 32 result_file.write "\n"
33 33 result_file.close
34 34 exit(0)
35 35 }
36 36
37 37 report_wrong = lambda {
38 38 result_file.write "Incorrect\n"
39 39 result_file.write "0\n"
40 40 result_file.close
41 41 exit(0)
42 42 }
43 43
44 44 ##################
45 45 # Your code here #
46 46 ##################
47 47
48 48 ########### THIS IS FOR CHECKING TEXT ##########
49 49
50 50 # check visible text
51 51
52 52 out_items = output_file_content.split
53 53 ans_items = answer_file_content.split
54 54
55 55 if out_items.length != ans_items.length
56 56 report_wrong.call
57 57 else
58 58 out_items.length.times do |i|
59 59 if out_items[i]!=ans_items[i]
60 60 report_wrong.call
61 61 end
62 62 end
63 63 report_correct.call
64 64 end
Show file before | Show file after | Raw diff | Download diff | Ignore whitespace Increase context Show commentsHide comments
@@ -1,47 +1,47
1 - #!/usr/bin/ruby
1 + #!/usr/bin/env ruby
2 2
3 3 problem_home = ENV['PROBLEM_HOME']
4 4 require "#{problem_home}/script/test_dsl.rb"
5 5
6 6 if ARGV.length < 2
7 7 puts "Usage: check <language> <test-number> [<output-file>]"
8 8 exit(0)
9 9 end
10 10
11 11 language = ARGV[0]
12 12 test_num = ARGV[1].to_i
13 13 if ARGV.length >= 3
14 14 output_file_name = ARGV[2]
15 15 else
16 16 output_file_name = "output.txt"
17 17 end
18 18
19 19 load "#{problem_home}/test_cases/all_tests.cfg"
20 20 problem = Problem.get_instance
21 21
22 22 output_file = File.new(output_file_name, "r")
23 23 answer_file = File.new("#{problem_home}/test_cases/#{test_num}/answer-#{test_num}.txt")
24 24 result_file = File.new("check_result", "w")
25 25
26 26 output_file_content = output_file.read
27 27 answer_file_content = answer_file.read
28 28
29 29 report_correct = lambda {
30 30 result_file.write "Correct\n"
31 31 result_file.write problem.get_score(test_num)
32 32 result_file.write "\n"
33 33 result_file.close
34 34 exit(0)
35 35 }
36 36
37 37 report_wrong = lambda {
38 38 result_file.write "Incorrect\n"
39 39 result_file.write "0\n"
40 40 result_file.close
41 41 exit(0)
42 42 }
43 43
44 44 ##################
45 45 # Your code here #
46 46 ##################
47 47 report_correct.call
Show file before | Show file after | Raw diff | Download diff | Ignore whitespace Increase context Show commentsHide comments
@@ -109,220 +109,220
109 109 end
110 110
111 111 it "should not allow malicious submission to see PROBLEM_HOME" do
112 112 problem_test_yesno = stub(Problem,
113 113 :id => 1, :name => 'test_yesno',
114 114 :full_score => 10)
115 115 grader_should(:grade => "yesno_access_problem_home.c",
116 116 :on => problem_test_yesno,
117 117 :and_report => {
118 118 :score => 0,
119 119 :comment => /(-|x)/})
120 120 end
121 121
122 122 it "should not allow malicious submission to open files" do
123 123 problem_test_yesno = stub(Problem,
124 124 :id => 1, :name => 'test_yesno',
125 125 :full_score => 10)
126 126 grader_should(:grade => "yesno_open_file.c",
127 127 :on => problem_test_yesno,
128 128 :and_report => {
129 129 :score => 0,
130 130 :comment => /(-|x)/})
131 131 end
132 132
133 133 def grader_should(args)
134 134 @user1 = stub(User,
135 135 :id => 1, :login => 'user1')
136 136
137 137 submission =
138 138 create_submission_from_file(1, @user1, args[:on], args[:grade], args[:language])
139 139 submission.should_receive(:graded_at=)
140 140
141 141 expected_score = args[:and_report][:score]
142 142 expected_comment = args[:and_report][:comment]
143 143 if args[:and_report][:compiler_message]!=nil
144 144 expected_compiler_message = args[:and_report][:compiler_message]
145 145 else
146 146 expected_compiler_message = ''
147 147 end
148 148
149 149 submission.should_receive(:points=).with(expected_score)
150 150 submission.should_receive(:grader_comment=).with(expected_comment)
151 151 submission.should_receive(:compiler_message=).with(expected_compiler_message)
152 152 submission.should_receive(:save)
153 153
154 154 @engine.grade(submission)
155 155 end
156 156
157 157 protected
158 158
159 159 def create_normal_submission_mock_from_file(source_fname)
160 160 create_submission_from_file(1, @user_user1, @problem_test_normal, source_fname)
161 161 end
162 162
163 163 end
164 164
165 165 describe "A grader engine, when grading test requests" do
166 166
167 167 include GraderEngineHelperMethods
168 168
169 169 before(:each) do
170 170 @config = Grader::Configuration.get_instance
171 171 @engine = Grader::Engine.new(:room_maker => Grader::TestRequestRoomMaker.new,
172 172 :reporter => Grader::TestRequestReporter.new)
173 173 init_sandbox
174 174 end
175 175
176 176 it "should report error if there is no problem template" do
177 177 problem = stub(Problem,
178 178 :id => 1, :name => 'nothing')
179 179 grader_should(:grade => 'test1_correct.c',
180 180 :on => problem,
181 181 :with => 'in1.txt',
182 182 :and_report => {
183 183 :graded_at= => nil,
184 184 :compiler_message= => '',
185 185 :grader_comment= => '',
186 186 :running_stat= => /template not found/,
187 187 :running_time= => nil,
188 188 :exit_status= => nil,
189 189 :memory_usage= => nil,
190 190 :save => nil})
191 191 end
192 192
193 193 it "should run test request and produce output file" do
194 194 problem = stub(Problem,
195 195 :id => 1, :name => 'test_normal')
196 196 grader_should(:grade => 'test1_correct.c',
197 197 :on => problem,
198 198 :with => 'in1.txt',
199 199 :and_report => {
200 200 :graded_at= => nil,
201 201 :compiler_message= => '',
202 202 :grader_comment= => '',
203 203 :running_stat= => /0.0\d* sec./,
204 204 :output_file_name= => lambda { |fname|
205 - File.exists?(fname).should be_true
205 + File.exists?(fname).should be true
206 206 },
207 207 :running_time= => nil,
208 208 :exit_status= => nil,
209 209 :memory_usage= => nil,
210 210 :save => nil})
211 211 end
212 212
213 213 it "should clean up problem directory after running test request" do
214 214 problem = stub(Problem,
215 215 :id => 1, :name => 'test_normal')
216 216 grader_should(:grade => 'test1_correct.c',
217 217 :on => problem,
218 218 :with => 'in1.txt',
219 219 :and_report => {
220 220 :graded_at= => nil,
221 221 :compiler_message= => '',
222 222 :grader_comment= => '',
223 223 :running_stat= => nil,
224 224 :output_file_name= => nil,
225 225 :running_time= => nil,
226 226 :exit_status= => nil,
227 227 :memory_usage= => nil,
228 228 :save => nil})
229 - File.exists?(@config.user_result_dir + "/test_request/test_normal/test_cases/1/input-1.txt").should be_false
229 + File.exists?(@config.user_result_dir + "/test_request/test_normal/test_cases/1/input-1.txt").should be false
230 230 end
231 231
232 232 it "should compile test request with error and report compilation error" do
233 233 problem = stub(Problem,
234 234 :id => 1, :name => 'test_normal')
235 235 grader_should(:grade => 'test1_compile_error.c',
236 236 :on => problem,
237 237 :with => 'in1.txt',
238 238 :and_report => {
239 239 :graded_at= => nil,
240 240 :compiler_message= => /.+/,
241 241 :grader_comment= => /[Cc]ompil.*error/,
242 242 :running_stat= => '',
243 243 :save => nil})
244 244 end
245 245
246 246 it "should report exit status" do
247 247 problem = stub(Problem,
248 248 :id => 1, :name => 'test_normal')
249 249 grader_should(:grade => 'add_nonzero_exit_status.c',
250 250 :on => problem,
251 251 :with => 'in1.txt',
252 252 :and_report => {
253 253 :graded_at= => nil,
254 254 :compiler_message= => '',
255 255 :grader_comment= => '',
256 256 :running_stat= => /[Ee]xit.*status.*10.*0\.0\d* sec/m,
257 257 :output_file_name= => lambda { |fname|
258 - File.exists?(fname).should be_true
258 + File.exists?(fname).should be true
259 259 },
260 260 :running_time= => nil,
261 261 :exit_status= => /10/,
262 262 :memory_usage= => nil,
263 263 :save => nil})
264 264 end
265 265
266 266 it "should produce running statistics for normal submission" do
267 267 problem = stub(Problem,
268 268 :id => 1, :name => 'test_normal')
269 269 grader_should(:grade => 'test_run_stat.c',
270 270 :on => problem,
271 271 :with => 'in1.txt',
272 272 :and_report => {
273 273 :graded_at= => nil,
274 274 :compiler_message= => '',
275 275 :grader_comment= => '',
276 276 :running_stat= => nil,
277 277 :output_file_name= => lambda { |fname|
278 - File.exists?(fname).should be_true
278 + File.exists?(fname).should be true
279 279 },
280 280 :running_time= => lambda { |t|
281 281 (t>=0.14) and (t<=0.16)
282 282 },
283 283 :exit_status= => nil,
284 284 :memory_usage= => lambda { |s|
285 285 (s>=500) and (s<=1000)
286 286 },
287 287 :save => nil})
288 288 end
289 289
290 290 protected
291 291 def grader_should(args)
292 292 @user1 = stub(User,
293 293 :id => 1, :login => 'user1')
294 294
295 295 problem = args[:on]
296 296 input_file = @config.test_request_input_base_dir + "/" + args[:with]
297 297
298 298 submission =
299 299 create_submission_from_file(1, @user1, args[:on], args[:grade])
300 300
301 301 test_request = stub(TestRequest,
302 302 :id => 1,
303 303 :user => @user1,
304 304 :problem => problem,
305 305 :submission => submission,
306 306 :input_file_name => input_file,
307 307 :language => submission.language,
308 308 :problem_name => problem.name)
309 309
310 310 expectations = args[:and_report]
311 311
312 312 expectations.each do |key,val|
313 313 if val==nil
314 314 test_request.should_receive(key)
315 315 elsif val.class == Proc
316 316 test_request.should_receive(key) { |fname|
317 317 val.call(fname)
318 318 }
319 319 else
320 320 test_request.should_receive(key).with(val)
321 321 end
322 322 end
323 323
324 324 @engine.grade(test_request)
325 325 end
326 326
327 327 end
328 328
You need to be logged in to leave comments. Login now