Description:
[grader] fixed: import_problem (error in erb calls), check_wrapper; better error handling -- will get 0 score for a particular test run that fails
git-svn-id: http://theory.cpe.ku.ac.th/grader/judge/trunk/scripts@118 6386c4cd-e34a-4fa8-8920-d93eb39b512e
Commit status:
[Not Reviewed]
References:
Diff options:
Comments:
0 Commit comments
0 Inline Comments
Unresolved TODOs:
There are no unresolved TODOs
Author
jittat
-
r31:d810d6c2e99e - - 5 files changed: 28 inserted, 15 deleted
| @@ -81,53 +81,53 | |||
|
|
81 | 81 | puts "copying testcases" |
|
|
82 | 82 | |
|
|
83 | 83 | 1.upto(num_testcases) do |i| |
|
|
84 | 84 | system("mkdir #{problem}/test_cases/#{i}") |
|
|
85 | 85 | copy_testcase("#{testcase_dir}","#{problem}/test_cases/#{i}",i) |
|
|
86 | 86 | end |
|
|
87 | 87 | |
|
|
88 | 88 | |
|
|
89 | 89 | # generating all_tests.cfg |
|
|
90 | 90 | puts "generating testcase config file" |
|
|
91 | 91 | |
|
|
92 | 92 | template = File.open(SCRIPT_DIR + "/templates/all_tests.cfg.erb").read |
|
|
93 | 93 | all_test_cfg = ERB.new(template) |
|
|
94 | 94 | |
|
|
95 | 95 | cfg_file = File.open("#{problem}/test_cases/all_tests.cfg","w") |
|
|
96 | 96 | cfg_file.puts all_test_cfg.result |
|
|
97 | 97 | cfg_file.close |
|
|
98 | 98 | |
|
|
99 | 99 | |
|
|
100 | 100 | # copy check script |
|
|
101 | 101 | if res = /^wrapper:(.*)$/.match(check_script) |
|
|
102 | 102 | # wrapper script |
|
|
103 | 103 | check_script_fname = res[1] |
|
|
104 | 104 | script_name = File.basename(check_script_fname) |
|
|
105 | - check_wrapper_template = File.open(SCRIPT_DIR + "/templates/check_wrapper") | |
|
|
106 | - check_wrapper = ERB.new(template) | |
|
|
105 | + check_wrapper_template = File.open(SCRIPT_DIR + "/templates/check_wrapper").read | |
|
|
106 | + check_wrapper = ERB.new(check_wrapper_template) | |
|
|
107 | 107 | |
|
|
108 | 108 | check_file = File.open("#{problem}/script/check","w") |
|
|
109 | - check_file.puts check_wrapper | |
|
|
109 | + check_file.puts check_wrapper.result | |
|
|
110 | 110 | check_file.close |
|
|
111 | 111 | |
|
|
112 | 112 | File.chmod(0755,"#{problem}/script/check") |
|
|
113 | 113 | |
|
|
114 | 114 | system("cp #{check_script_fname} #{problem}/script/#{script_name}") |
|
|
115 | 115 | else |
|
|
116 | 116 | if File.exists?(SCRIPT_DIR + "/templates/check.#{check_script}") |
|
|
117 | 117 | check_script_fname = SCRIPT_DIR + "/templates/check.#{check_script}" |
|
|
118 | 118 | else |
|
|
119 | 119 | check_script_fname = check_script |
|
|
120 | 120 | end |
|
|
121 | 121 | system("cp #{check_script_fname} #{problem}/script/check") |
|
|
122 | 122 | end |
|
|
123 | 123 | |
|
|
124 | 124 | # generating test_request directory |
|
|
125 | 125 | puts "generating test_request template" |
|
|
126 | 126 | FileUtils.mkdir_p("test_request/#{problem}/script") |
|
|
127 | 127 | FileUtils.mkdir_p("test_request/#{problem}/test_cases/1") |
|
|
128 | 128 | |
|
|
129 | 129 | template = File.open(SCRIPT_DIR + "/templates/test_request_all_tests.cfg.erb").read |
|
|
130 | 130 | test_request_all_test_cfg = ERB.new(template) |
|
|
131 | 131 | |
|
|
132 | 132 | cfg_file = File.open("test_request/#{problem}/test_cases/all_tests.cfg","w") |
|
|
133 | 133 | cfg_file.puts test_request_all_test_cfg.result |
| @@ -1,73 +1,78 | |||
|
|
1 | 1 | #!/usr/bin/ruby |
|
|
2 | 2 | |
|
|
3 | + CORRECT_MARK = 'P' | |
|
|
4 | + INCORRECT_MARK = '-' | |
|
|
5 | + TIMEOUT_MARK = 'T' | |
|
|
6 | + RUN_ERROR_MARK = 'x' | |
|
|
7 | + | |
|
|
3 | 8 | def log(str='') |
|
|
4 | 9 | if ENV['TALKATIVE']!=nil |
|
|
5 | 10 | puts str |
|
|
6 | 11 | end |
|
|
7 | 12 | if ENV['GRADER_LOGGING']!=nil |
|
|
8 | 13 | log_fname = ENV['GRADER_LOGGING'] |
|
|
9 | 14 | fp = File.open(log_fname,"a") |
|
|
10 | 15 | fp.puts("grade: #{Time.new.strftime("%H:%M")} #{str}") |
|
|
11 | 16 | fp.close |
|
|
12 | 17 | end |
|
|
13 | 18 | end |
|
|
14 | 19 | |
|
|
15 | 20 | def char_comment(comment) |
|
|
16 | 21 | if comment =~ /[iI]ncorrect/ |
|
|
17 | - '-' | |
|
|
22 | + INCORRECT_MARK | |
|
|
18 | 23 | elsif comment =~ /[Cc]orrect/ |
|
|
19 | - 'P' | |
|
|
24 | + CORRECT_MARK | |
|
|
20 | 25 | elsif comment =~ /[Tt]ime/ |
|
|
21 | - 'T' | |
|
|
26 | + TIMEOUT_MARK | |
|
|
22 | 27 | else |
|
|
23 |
- |
|
|
|
28 | + RUN_ERROR_MARK # these are run time errors | |
|
|
24 | 29 | end |
|
|
25 | 30 | end |
|
|
26 | 31 | |
|
|
27 | 32 | problem_home = ENV['PROBLEM_HOME'] |
|
|
28 | 33 | require "#{problem_home}/script/test_dsl.rb" |
|
|
29 | 34 | load "#{problem_home}/test_cases/all_tests.cfg" |
|
|
30 | 35 | problem = Problem.get_instance |
|
|
31 | 36 | |
|
|
32 | 37 | if problem.well_formed? == false |
|
|
33 | 38 | log "The problem specification is not well formed." |
|
|
34 | 39 | exit(127) |
|
|
35 | 40 | end |
|
|
36 | 41 | |
|
|
37 | 42 | all_score = 0 |
|
|
38 | 43 | all_comment = '' |
|
|
39 | 44 | (1..(problem.runs.length-1)).each do |k| |
|
|
40 | 45 | log "grade run #{k}" |
|
|
41 | 46 | run = problem.runs[k] |
|
|
42 | 47 | run_score = 0 |
|
|
43 | 48 | run_comment = '' |
|
|
44 | 49 | run_comment_short = '' |
|
|
45 | 50 | run.tests.each do |test_num| |
|
|
46 | 51 | result_file_name = "#{test_num}/result" |
|
|
47 | 52 | if not File.exists?(result_file_name) |
|
|
48 | 53 | run_comment += "result file for test #{test_num} not found\n" |
|
|
49 |
- run_comment_short += |
|
|
|
54 | + run_comment_short += RUN_ERROR_MARK | |
|
|
50 | 55 | log "Cannot find the file #{test_num}/result!" |
|
|
51 | 56 | else |
|
|
52 | 57 | result_file = File.new(result_file_name, "r") |
|
|
53 | 58 | result_file_lines = result_file.readlines |
|
|
54 | 59 | run_score = run_score + result_file_lines[1].to_i |
|
|
55 | 60 | run_comment += result_file_lines[0] |
|
|
56 | 61 | run_comment_short += char_comment(result_file_lines[0]) |
|
|
57 | 62 | result_file.close |
|
|
58 | 63 | end |
|
|
59 | 64 | end |
|
|
60 | 65 | |
|
|
61 | 66 | run_result_file = File.new("result-#{k}", "w") |
|
|
62 | 67 | run_result_file.write run_score |
|
|
63 | 68 | run_result_file.write "\n" |
|
|
64 | 69 | run_result_file.close |
|
|
65 | 70 | |
|
|
66 | 71 | run_comment_file = File.new("comment-#{k}", "w") |
|
|
67 | 72 | run_comment_file.write "#{run_comment}\n" |
|
|
68 | 73 | run_comment_file.close |
|
|
69 | 74 | |
|
|
70 | 75 | all_score = all_score + run_score |
|
|
71 | 76 | all_comment += run_comment_short |
|
|
72 | 77 | end |
|
|
73 | 78 | |
| @@ -1,45 +1,45 | |||
|
|
1 | 1 | #!/usr/bin/ruby |
|
|
2 | 2 | |
|
|
3 | 3 | def log(str='') |
|
|
4 | 4 | if ENV['TALKATIVE']!=nil |
|
|
5 | 5 | puts str |
|
|
6 | 6 | end |
|
|
7 | 7 | if ENV['GRADER_LOGGING']!=nil |
|
|
8 | 8 | log_fname = ENV['GRADER_LOGGING'] |
|
|
9 | 9 | fp = File.open(log_fname,"a") |
|
|
10 | 10 | fp.puts("judge: #{Time.new.strftime("%H:%M")} #{str}") |
|
|
11 | 11 | fp.close |
|
|
12 | 12 | end |
|
|
13 | 13 | end |
|
|
14 | 14 | |
|
|
15 | 15 | problem_home = ENV['PROBLEM_HOME'] |
|
|
16 | 16 | |
|
|
17 | 17 | def execute(command, error_message="") |
|
|
18 | 18 | if not system(command) |
|
|
19 |
- |
|
|
|
20 | - puts "ERROR: #{error_message}" | |
|
|
21 | - exit(127) | |
|
|
19 | + msg = "ERROR: #{error_message}" | |
|
|
20 | + log msg | |
|
|
21 | + raise msg | |
|
|
22 | 22 | end |
|
|
23 | 23 | end |
|
|
24 | 24 | |
|
|
25 | 25 | # ARGV[0] --- language |
|
|
26 | 26 | # ARGV[1] --- program source file |
|
|
27 | 27 | # ARGV[2] --- test result directory |
|
|
28 | 28 | # ARGV[3] --- sandbox directory |
|
|
29 | 29 | |
|
|
30 | 30 | if ARGV.length < 2 || ARGV.length > 4 |
|
|
31 | 31 | puts "Usage: judge <language> <program-source> [<test-result-directory>] [<sandbox-directory>]" |
|
|
32 | 32 | puts " <sandbox-directory> is defaulted to ./sandbox" |
|
|
33 | 33 | puts " <test-result-directory> is defaulted to ./test-result" |
|
|
34 | 34 | puts "WARNING: The judge script will forcefully create the (implicitly and explicitly) specified directories and remove anything inside it." |
|
|
35 | 35 | exit(127) |
|
|
36 | 36 | end |
|
|
37 | 37 | |
|
|
38 | 38 | language = ARGV[0] |
|
|
39 | 39 | if language != "c" && language != "c++" && language != "pascal" |
|
|
40 | 40 | log "You specified a language that is not supported." |
|
|
41 | 41 | exit(127) |
|
|
42 | 42 | end |
|
|
43 | 43 | |
|
|
44 | 44 | source_file = ARGV[1] |
|
|
45 | 45 | if File.exist?(source_file) == false |
| @@ -84,45 +84,49 | |||
|
|
84 | 84 | compile_message = `cat compiler_message` |
|
|
85 | 85 | compile_message.strip! |
|
|
86 | 86 | execute("mv compiler_message #{test_result_dir}", "Cannot move the compiler message to #{test_result_dir}.") |
|
|
87 | 87 | if !FileTest.exist?("a.out") |
|
|
88 | 88 | log "Cannot compile the source code. See message in #{test_result_dir}/compile_message" |
|
|
89 | 89 | exit(127) |
|
|
90 | 90 | else |
|
|
91 | 91 | execute("mv a.out #{test_result_dir}", "Cannot move the compiled program to #{test_result_dir}") |
|
|
92 | 92 | system("rm -Rf #{sandbox_dir}/*") |
|
|
93 | 93 | end |
|
|
94 | 94 | |
|
|
95 | 95 | require "#{problem_home}/script/test_dsl.rb" |
|
|
96 | 96 | load "#{problem_home}/test_cases/all_tests.cfg" |
|
|
97 | 97 | problem = Problem.get_instance |
|
|
98 | 98 | |
|
|
99 | 99 | if problem.well_formed? == false |
|
|
100 | 100 | log "The problem specification is not well formed." |
|
|
101 | 101 | exit(127) |
|
|
102 | 102 | end |
|
|
103 | 103 | |
|
|
104 | 104 | # Doing the testing. |
|
|
105 | 105 | (1..(problem.num_tests)).each do |test_num| |
|
|
106 | 106 | log "Test number: #{test_num}" |
|
|
107 | 107 | execute("cp #{test_result_dir}/a.out #{sandbox_dir}", "Cannot copy the compiled program into #{sandbox_dir}") |
|
|
108 | - execute("#{problem_home}/script/run #{language} #{test_num}", "Error occured during execution of the run script") | |
|
|
108 | + begin | |
|
|
109 | + execute("#{problem_home}/script/run #{language} #{test_num}", "Error occured during execution of the run script") | |
|
|
110 | + rescue | |
|
|
111 | + # do nothing | |
|
|
112 | + end | |
|
|
109 | 113 | execute("mkdir #{test_result_dir}/#{test_num}", "Cannot create directory #{test_result_dir}/#{test_num}") |
|
|
110 | 114 | execute("mv #{sandbox_dir}/result #{test_result_dir}/#{test_num}", "Cannot copy the result file into #{test_result_dir}/#{test_num}") |
|
|
111 | 115 | execute("mv #{sandbox_dir}/comment #{test_result_dir}/#{test_num}", "Cannot copy the comment file into #{test_result_dir}/#{test_num}") |
|
|
112 | 116 | execute("mv #{sandbox_dir}/output.txt #{test_result_dir}/#{test_num}", "Cannot copy the output file into #{test_result_dir}/#{test_num}") |
|
|
113 | 117 | execute("rm -Rf #{sandbox_dir}/*", "Cannot clear #{sandbox_dir}") |
|
|
114 | 118 | end |
|
|
115 | 119 | |
|
|
116 | 120 | # Grade |
|
|
117 | 121 | log |
|
|
118 | 122 | log "Grading..." |
|
|
119 | 123 | begin |
|
|
120 | 124 | Dir.chdir test_result_dir |
|
|
121 | 125 | rescue |
|
|
122 | 126 | log "ERROR: Cannot change directory to #{test_result_dir}." |
|
|
123 | 127 | exit(127) |
|
|
124 | 128 | end |
|
|
125 | 129 | execute("#{problem_home}/script/grade", "An error occured during grading!") |
|
|
126 | 130 | |
|
|
127 | 131 | log |
|
|
128 | 132 | log "All done!" |
| @@ -112,31 +112,32 | |||
|
|
112 | 112 | end |
|
|
113 | 113 | comment_file.close |
|
|
114 | 114 | |
|
|
115 | 115 | log "Done!" |
|
|
116 | 116 | exit(0) |
|
|
117 | 117 | } |
|
|
118 | 118 | |
|
|
119 | 119 | if run_result[0][0,2] != "OK" |
|
|
120 | 120 | log "There was a runtime error." |
|
|
121 | 121 | report.call(run_result[0], 0, "No comment.\n") |
|
|
122 | 122 | end |
|
|
123 | 123 | |
|
|
124 | 124 | if running_time[:user].to_f + running_time[:sys].to_f > time_limit |
|
|
125 | 125 | log "Time limit exceeded." |
|
|
126 | 126 | report.call("Time limit exceeded", 0, "No comment.\n") |
|
|
127 | 127 | end |
|
|
128 | 128 | |
|
|
129 | 129 | # Run 'check' to evaluate the output. |
|
|
130 | 130 | #puts "There was no runtime error. Proceed to checking the output." |
|
|
131 | 131 | check_command = "#{problem_home}/script/check #{language} #{test_num}" |
|
|
132 | 132 | log "Checking the output..." |
|
|
133 | 133 | log check_command |
|
|
134 | 134 | if not system(check_command) |
|
|
135 | 135 | log "Problem with check script" |
|
|
136 | + report.call("Incorrect",0,"Check script error.\n") | |
|
|
136 | 137 | exit(127) |
|
|
137 | 138 | end |
|
|
138 | 139 | |
|
|
139 | 140 | check_file = File.new("check_result", "r") |
|
|
140 | 141 | check_file_lines = check_file.readlines |
|
|
141 | 142 | |
|
|
142 | 143 | report.call(check_file_lines[0], check_file_lines[1], "No comment.\n") |
| @@ -13,33 +13,36 | |||
|
|
13 | 13 | # (script) <lang> <test-num> <in-file> <out-file> <ans-file> <full-score> |
|
|
14 | 14 | # |
|
|
15 | 15 | # and REAL_CHECK_SCRIPT's output to standard out is redirected to |
|
|
16 | 16 | # 'check_result' as required by normal check script. |
|
|
17 | 17 | |
|
|
18 | 18 | problem_home = ENV['PROBLEM_HOME'] |
|
|
19 | 19 | require "#{problem_home}/script/test_dsl.rb" |
|
|
20 | 20 | |
|
|
21 | 21 | if ARGV.length < 2 |
|
|
22 | 22 | puts "Usage: check <language> <test-number> [<output-file>]" |
|
|
23 | 23 | exit(0) |
|
|
24 | 24 | end |
|
|
25 | 25 | |
|
|
26 | 26 | language = ARGV[0] |
|
|
27 | 27 | test_num = ARGV[1].to_i |
|
|
28 | 28 | if ARGV.length >= 3 |
|
|
29 | 29 | output_file_name = ARGV[2] |
|
|
30 | 30 | else |
|
|
31 | 31 | output_file_name = "output.txt" |
|
|
32 | 32 | end |
|
|
33 | 33 | |
|
|
34 | 34 | load "#{problem_home}/test_cases/all_tests.cfg" |
|
|
35 | 35 | problem = Problem.get_instance |
|
|
36 | 36 | |
|
|
37 |
- answer_file_name = |
|
|
|
38 |
- input_file_name = |
|
|
|
37 | + answer_file_name = "#{problem_home}/test_cases/#{test_num}/answer-#{test_num}.txt" | |
|
|
38 | + input_file_name = "#{problem_home}/test_cases/#{test_num}/input-#{test_num}.txt" | |
|
|
39 | 39 | |
|
|
40 |
- score = |
|
|
|
40 | + score = problem.get_score(test_num) | |
|
|
41 | 41 | |
|
|
42 | 42 | cmd = "#{problem_home}/script/#{REAL_CHECK_SCRIPT} " + |
|
|
43 | 43 | "#{language} #{test_num} #{input_file_name} #{output_file_name} " + |
|
|
44 | 44 | "#{answer_file_name} #{score} > check_result" |
|
|
45 | 45 | |
|
|
46 | + #puts "wrapper-CMD: #{cmd}" | |
|
|
47 | + | |
|
|
48 | + system(cmd) | |
You need to be logged in to leave comments.
Login now