cafe-grader-judge-scripts

Commit r31:d810d6c2e99e public

parent | child
Description:
[grader] fixed: import_problem (error in erb calls), check_wrapper; better error handling -- will get 0 score for a particular test run that fails git-svn-id: http://theory.cpe.ku.ac.th/grader/judge/trunk/scripts@118 6386c4cd-e34a-4fa8-8920-d93eb39b512e
Commit status:
[Not Reviewed]
References:
default
Diff options:
Raw Diff | Patch Diff | Download Diff | Show whitespace | Increase context
Comments:
0 Commit comments 0 Inline Comments
Unresolved TODOs:
There are no unresolved TODOs
Side by Side Unified
Expand All Files Collapse All Files Wide Mode Diff
Add another comment
Browse Files

r31:d810d6c2e99e - - 5 files changed: 27 inserted, 14 deleted

Show file before | Show file after | Raw diff | Download diff | Show whitespace Increase context Show commentsHide comments
@@ -57,83 +57,83
57 57 * creates a test_request template in the current directory + '/test_request'
58 58
59 59 For wrapped checked script see comment in templates/check_wrapper for
60 60 information.
61 61 USAGE
62 62 exit(127)
63 63 end
64 64
65 65 # processing arguments
66 66 problem = ARGV[0]
67 67 testcase_dir = ARGV[1]
68 68 num_testcases = ARGV[2].to_i
69 69 check_script = ARGV[3]
70 70 options = {:time_limit => 1, :mem_limit => 16}
71 71 process_options(options)
72 72
73 73 # start working
74 74 puts "creating directories"
75 75
76 76 system("mkdir #{problem}")
77 77 system("mkdir #{problem}/script")
78 78 system("mkdir #{problem}/test_cases")
79 79 #system("cp #{GRADER_DIR}/std-script/* #{problem}/script")
80 80
81 81 puts "copying testcases"
82 82
83 83 1.upto(num_testcases) do |i|
84 84 system("mkdir #{problem}/test_cases/#{i}")
85 85 copy_testcase("#{testcase_dir}","#{problem}/test_cases/#{i}",i)
86 86 end
87 87
88 88
89 89 # generating all_tests.cfg
90 90 puts "generating testcase config file"
91 91
92 92 template = File.open(SCRIPT_DIR + "/templates/all_tests.cfg.erb").read
93 93 all_test_cfg = ERB.new(template)
94 94
95 95 cfg_file = File.open("#{problem}/test_cases/all_tests.cfg","w")
96 96 cfg_file.puts all_test_cfg.result
97 97 cfg_file.close
98 98
99 99
100 100 # copy check script
101 101 if res = /^wrapper:(.*)$/.match(check_script)
102 102 # wrapper script
103 103 check_script_fname = res[1]
104 104 script_name = File.basename(check_script_fname)
105 - check_wrapper_template = File.open(SCRIPT_DIR + "/templates/check_wrapper")
106 - check_wrapper = ERB.new(template)
105 + check_wrapper_template = File.open(SCRIPT_DIR + "/templates/check_wrapper").read
106 + check_wrapper = ERB.new(check_wrapper_template)
107 107
108 108 check_file = File.open("#{problem}/script/check","w")
109 - check_file.puts check_wrapper
109 + check_file.puts check_wrapper.result
110 110 check_file.close
111 111
112 112 File.chmod(0755,"#{problem}/script/check")
113 113
114 114 system("cp #{check_script_fname} #{problem}/script/#{script_name}")
115 115 else
116 116 if File.exists?(SCRIPT_DIR + "/templates/check.#{check_script}")
117 117 check_script_fname = SCRIPT_DIR + "/templates/check.#{check_script}"
118 118 else
119 119 check_script_fname = check_script
120 120 end
121 121 system("cp #{check_script_fname} #{problem}/script/check")
122 122 end
123 123
124 124 # generating test_request directory
125 125 puts "generating test_request template"
126 126 FileUtils.mkdir_p("test_request/#{problem}/script")
127 127 FileUtils.mkdir_p("test_request/#{problem}/test_cases/1")
128 128
129 129 template = File.open(SCRIPT_DIR + "/templates/test_request_all_tests.cfg.erb").read
130 130 test_request_all_test_cfg = ERB.new(template)
131 131
132 132 cfg_file = File.open("test_request/#{problem}/test_cases/all_tests.cfg","w")
133 133 cfg_file.puts test_request_all_test_cfg.result
134 134 cfg_file.close
135 135
136 136 system("cp #{SCRIPT_DIR}/templates/check_empty test_request/#{problem}/script/check")
137 137 system("cp #{SCRIPT_DIR}/templates/answer-1.txt test_request/#{problem}/test_cases/1")
138 138
139 139 puts "done"
Show file before | Show file after | Raw diff | Download diff | Show whitespace Increase context Show commentsHide comments
@@ -1,81 +1,86
1 1 #!/usr/bin/ruby
2 2
3 + CORRECT_MARK = 'P'
4 + INCORRECT_MARK = '-'
5 + TIMEOUT_MARK = 'T'
6 + RUN_ERROR_MARK = 'x'
7 +
3 8 def log(str='')
4 9 if ENV['TALKATIVE']!=nil
5 10 puts str
6 11 end
7 12 if ENV['GRADER_LOGGING']!=nil
8 13 log_fname = ENV['GRADER_LOGGING']
9 14 fp = File.open(log_fname,"a")
10 15 fp.puts("grade: #{Time.new.strftime("%H:%M")} #{str}")
11 16 fp.close
12 17 end
13 18 end
14 19
15 20 def char_comment(comment)
16 21 if comment =~ /[iI]ncorrect/
17 - '-'
22 + INCORRECT_MARK
18 23 elsif comment =~ /[Cc]orrect/
19 - 'P'
24 + CORRECT_MARK
20 25 elsif comment =~ /[Tt]ime/
21 - 'T'
26 + TIMEOUT_MARK
22 27 else
23 - 'x' # these are run time errors
28 + RUN_ERROR_MARK # these are run time errors
24 29 end
25 30 end
26 31
27 32 problem_home = ENV['PROBLEM_HOME']
28 33 require "#{problem_home}/script/test_dsl.rb"
29 34 load "#{problem_home}/test_cases/all_tests.cfg"
30 35 problem = Problem.get_instance
31 36
32 37 if problem.well_formed? == false
33 38 log "The problem specification is not well formed."
34 39 exit(127)
35 40 end
36 41
37 42 all_score = 0
38 43 all_comment = ''
39 44 (1..(problem.runs.length-1)).each do |k|
40 45 log "grade run #{k}"
41 46 run = problem.runs[k]
42 47 run_score = 0
43 48 run_comment = ''
44 49 run_comment_short = ''
45 50 run.tests.each do |test_num|
46 51 result_file_name = "#{test_num}/result"
47 52 if not File.exists?(result_file_name)
48 53 run_comment += "result file for test #{test_num} not found\n"
49 - run_comment_short += 'x'
54 + run_comment_short += RUN_ERROR_MARK
50 55 log "Cannot find the file #{test_num}/result!"
51 56 else
52 57 result_file = File.new(result_file_name, "r")
53 58 result_file_lines = result_file.readlines
54 59 run_score = run_score + result_file_lines[1].to_i
55 60 run_comment += result_file_lines[0]
56 61 run_comment_short += char_comment(result_file_lines[0])
57 62 result_file.close
58 63 end
59 64 end
60 65
61 66 run_result_file = File.new("result-#{k}", "w")
62 67 run_result_file.write run_score
63 68 run_result_file.write "\n"
64 69 run_result_file.close
65 70
66 71 run_comment_file = File.new("comment-#{k}", "w")
67 72 run_comment_file.write "#{run_comment}\n"
68 73 run_comment_file.close
69 74
70 75 all_score = all_score + run_score
71 76 all_comment += run_comment_short
72 77 end
73 78
74 79 result_file = File.new("result", "w")
75 80 result_file.write all_score
76 81 result_file.write "\n"
77 82 result_file.close
78 83
79 84 comment_file = File.new("comment", "w")
80 85 comment_file.write "#{all_comment}\n"
81 86 comment_file.close
Show file before | Show file after | Raw diff | Download diff | Show whitespace Increase context Show commentsHide comments
@@ -1,128 +1,132
1 1 #!/usr/bin/ruby
2 2
3 3 def log(str='')
4 4 if ENV['TALKATIVE']!=nil
5 5 puts str
6 6 end
7 7 if ENV['GRADER_LOGGING']!=nil
8 8 log_fname = ENV['GRADER_LOGGING']
9 9 fp = File.open(log_fname,"a")
10 10 fp.puts("judge: #{Time.new.strftime("%H:%M")} #{str}")
11 11 fp.close
12 12 end
13 13 end
14 14
15 15 problem_home = ENV['PROBLEM_HOME']
16 16
17 17 def execute(command, error_message="")
18 18 if not system(command)
19 - log "ERROR: #{error_message}"
20 - puts "ERROR: #{error_message}"
21 - exit(127)
19 + msg = "ERROR: #{error_message}"
20 + log msg
21 + raise msg
22 22 end
23 23 end
24 24
25 25 # ARGV[0] --- language
26 26 # ARGV[1] --- program source file
27 27 # ARGV[2] --- test result directory
28 28 # ARGV[3] --- sandbox directory
29 29
30 30 if ARGV.length < 2 || ARGV.length > 4
31 31 puts "Usage: judge <language> <program-source> [<test-result-directory>] [<sandbox-directory>]"
32 32 puts " <sandbox-directory> is defaulted to ./sandbox"
33 33 puts " <test-result-directory> is defaulted to ./test-result"
34 34 puts "WARNING: The judge script will forcefully create the (implicitly and explicitly) specified directories and remove anything inside it."
35 35 exit(127)
36 36 end
37 37
38 38 language = ARGV[0]
39 39 if language != "c" && language != "c++" && language != "pascal"
40 40 log "You specified a language that is not supported."
41 41 exit(127)
42 42 end
43 43
44 44 source_file = ARGV[1]
45 45 if File.exist?(source_file) == false
46 46 log "The source file does not exist."
47 47 exit(127)
48 48 end
49 49
50 50 log "Making test result and sandbox directories..."
51 51
52 52 current_dir = `pwd`
53 53 current_dir.strip!
54 54
55 55 if ARGV.length >= 3
56 56 test_result_dir = ARGV[2]
57 57 else
58 58 test_result_dir = "#{current_dir}/test-result"
59 59 end
60 60 log "Test result directory: #{test_result_dir}"
61 61 system("rm -Rf #{test_result_dir}")
62 62 execute("mkdir #{test_result_dir}", "Cannot make directory #{test_result_dir}.")
63 63
64 64 if ARGV.length >= 4
65 65 sandbox_dir = ARGV[3]
66 66 else
67 67 sandbox_dir = "#{current_dir}/sandbox"
68 68 end
69 69 log "Sandbox directory: #{sandbox_dir}"
70 70 system("rm -Rf #{sandbox_dir}")
71 71 execute("mkdir #{sandbox_dir}", "Cannot make directory #{sandbox_dir}")
72 72
73 73 # Compile
74 74 log
75 75 log "Compiling..."
76 76 execute("cp #{source_file} #{sandbox_dir}", "Cannot copy the source file to #{sandbox_dir}")
77 77 begin
78 78 Dir.chdir sandbox_dir
79 79 rescue
80 80 log "ERROR: Cannot change directory to #{sandbox_dir}."
81 81 exit(127)
82 82 end
83 83 execute("#{problem_home}/script/compile #{language} #{source_file}", "Compilation error!")
84 84 compile_message = `cat compiler_message`
85 85 compile_message.strip!
86 86 execute("mv compiler_message #{test_result_dir}", "Cannot move the compiler message to #{test_result_dir}.")
87 87 if !FileTest.exist?("a.out")
88 88 log "Cannot compile the source code. See message in #{test_result_dir}/compile_message"
89 89 exit(127)
90 90 else
91 91 execute("mv a.out #{test_result_dir}", "Cannot move the compiled program to #{test_result_dir}")
92 92 system("rm -Rf #{sandbox_dir}/*")
93 93 end
94 94
95 95 require "#{problem_home}/script/test_dsl.rb"
96 96 load "#{problem_home}/test_cases/all_tests.cfg"
97 97 problem = Problem.get_instance
98 98
99 99 if problem.well_formed? == false
100 100 log "The problem specification is not well formed."
101 101 exit(127)
102 102 end
103 103
104 104 # Doing the testing.
105 105 (1..(problem.num_tests)).each do |test_num|
106 106 log "Test number: #{test_num}"
107 107 execute("cp #{test_result_dir}/a.out #{sandbox_dir}", "Cannot copy the compiled program into #{sandbox_dir}")
108 + begin
108 109 execute("#{problem_home}/script/run #{language} #{test_num}", "Error occured during execution of the run script")
110 + rescue
111 + # do nothing
112 + end
109 113 execute("mkdir #{test_result_dir}/#{test_num}", "Cannot create directory #{test_result_dir}/#{test_num}")
110 114 execute("mv #{sandbox_dir}/result #{test_result_dir}/#{test_num}", "Cannot copy the result file into #{test_result_dir}/#{test_num}")
111 115 execute("mv #{sandbox_dir}/comment #{test_result_dir}/#{test_num}", "Cannot copy the comment file into #{test_result_dir}/#{test_num}")
112 116 execute("mv #{sandbox_dir}/output.txt #{test_result_dir}/#{test_num}", "Cannot copy the output file into #{test_result_dir}/#{test_num}")
113 117 execute("rm -Rf #{sandbox_dir}/*", "Cannot clear #{sandbox_dir}")
114 118 end
115 119
116 120 # Grade
117 121 log
118 122 log "Grading..."
119 123 begin
120 124 Dir.chdir test_result_dir
121 125 rescue
122 126 log "ERROR: Cannot change directory to #{test_result_dir}."
123 127 exit(127)
124 128 end
125 129 execute("#{problem_home}/script/grade", "An error occured during grading!")
126 130
127 131 log
128 132 log "All done!"
Show file before | Show file after | Raw diff | Download diff | Show whitespace Increase context Show commentsHide comments
@@ -88,55 +88,56
88 88 comment_file = File.new("comment", "w")
89 89
90 90 # Check if the program actually produced any output.
91 91 run_result_file = File.new("run_result", "r")
92 92 run_result = run_result_file.readlines
93 93 run_result_file.close
94 94 time_elapsed = run_result[run_result.length-1]
95 95 running_time = extract_time(time_elapsed)
96 96
97 97 report = lambda{ |status, points, comment|
98 98 result_file.write status.strip
99 99 result_file.write "\n"
100 100 result_file.write points.to_s.strip
101 101 result_file.write "\n"
102 102 result_file.write time_elapsed.strip
103 103 result_file.write "\n"
104 104 result_file.close
105 105 `rm run_result`
106 106 # `rm output.txt` --- keep the output
107 107
108 108 comment_file.write comment
109 109 comment_file.write "--run-result--\n"
110 110 run_result.each do |l|
111 111 comment_file.write l
112 112 end
113 113 comment_file.close
114 114
115 115 log "Done!"
116 116 exit(0)
117 117 }
118 118
119 119 if run_result[0][0,2] != "OK"
120 120 log "There was a runtime error."
121 121 report.call(run_result[0], 0, "No comment.\n")
122 122 end
123 123
124 124 if running_time[:user].to_f + running_time[:sys].to_f > time_limit
125 125 log "Time limit exceeded."
126 126 report.call("Time limit exceeded", 0, "No comment.\n")
127 127 end
128 128
129 129 # Run 'check' to evaluate the output.
130 130 #puts "There was no runtime error. Proceed to checking the output."
131 131 check_command = "#{problem_home}/script/check #{language} #{test_num}"
132 132 log "Checking the output..."
133 133 log check_command
134 134 if not system(check_command)
135 135 log "Problem with check script"
136 + report.call("Incorrect",0,"Check script error.\n")
136 137 exit(127)
137 138 end
138 139
139 140 check_file = File.new("check_result", "r")
140 141 check_file_lines = check_file.readlines
141 142
142 143 report.call(check_file_lines[0], check_file_lines[1], "No comment.\n")
Show file before | Show file after | Raw diff | Download diff | Show whitespace Increase context Show commentsHide comments
@@ -1,45 +1,48
1 1 #!/usr/bin/ruby
2 2
3 3 #
4 4 # This is a check script wrapper. It read all required information
5 5 # and call a real check script call REAL_CHECK_SCRIPT in directory
6 6 # [problem_home]/script
7 7 #
8 8
9 9 REAL_CHECK_SCRIPT = "<%= script_name %>"
10 10
11 11 # The REAL_CHECK_SCRIPT is called with:
12 12 #
13 13 # (script) <lang> <test-num> <in-file> <out-file> <ans-file> <full-score>
14 14 #
15 15 # and REAL_CHECK_SCRIPT's output to standard out is redirected to
16 16 # 'check_result' as required by normal check script.
17 17
18 18 problem_home = ENV['PROBLEM_HOME']
19 19 require "#{problem_home}/script/test_dsl.rb"
20 20
21 21 if ARGV.length < 2
22 22 puts "Usage: check <language> <test-number> [<output-file>]"
23 23 exit(0)
24 24 end
25 25
26 26 language = ARGV[0]
27 27 test_num = ARGV[1].to_i
28 28 if ARGV.length >= 3
29 29 output_file_name = ARGV[2]
30 30 else
31 31 output_file_name = "output.txt"
32 32 end
33 33
34 34 load "#{problem_home}/test_cases/all_tests.cfg"
35 35 problem = Problem.get_instance
36 36
37 - answer_file_name = File.new("#{problem_home}/test_cases/#{test_num}/answer-#{test_num}.txt")
38 - input_file_name = File.new("#{problem_home}/test_cases/#{test_num}/input-#{test_num}.txt")
37 + answer_file_name = "#{problem_home}/test_cases/#{test_num}/answer-#{test_num}.txt"
38 + input_file_name = "#{problem_home}/test_cases/#{test_num}/input-#{test_num}.txt"
39 39
40 - score = Problem.get_score(test_num)
40 + score = problem.get_score(test_num)
41 41
42 42 cmd = "#{problem_home}/script/#{REAL_CHECK_SCRIPT} " +
43 43 "#{language} #{test_num} #{input_file_name} #{output_file_name} " +
44 44 "#{answer_file_name} #{score} > check_result"
45 45
46 + #puts "wrapper-CMD: #{cmd}"
47 +
48 + system(cmd)
You need to be logged in to leave comments. Login now