cafe-grader-judge-scripts

Commit r111:06c23d391544 public

parent | child
Description:
added quick hack on running scripts, and compiler calls
Commit status:
[Not Reviewed]
References:
default
Diff options:
Raw Diff | Patch Diff | Download Diff | Show whitespace | Increase context
Comments:
0 Commit comments 0 Inline Comments
Unresolved TODOs:
There are no unresolved TODOs
Side by Side Unified
Expand All Files Collapse All Files Wide Mode Diff
Add another comment
Browse Files

r111:06c23d391544 - - 4 files changed: 12 inserted, 10 deleted

Show file before | Show file after | Raw diff | Download diff | Show whitespace Increase context Show commentsHide comments
@@ -1,187 +1,187
1 1 require 'fileutils'
2 2 require File.join(File.dirname(__FILE__),'dir_init')
3 3
4 4 module Grader
5 5
6 6 #
7 7 # A grader engine grades a submission, against anything: a test
8 8 # data, or a user submitted test data. It uses two helpers objects:
9 9 # room_maker and reporter.
10 10 #
11 11 class Engine
12 12
13 13 attr_writer :room_maker
14 14 attr_writer :reporter
15 15
16 16 def initialize(options={})
17 17 # default options
18 18 if not options.include? :room_maker
19 19 options[:room_maker] = Grader::SubmissionRoomMaker.new
20 20 end
21 21 if not options.include? :reporter
22 22 options[:reporter] = Grader::SubmissionReporter.new
23 23 end
24 24
25 25 @config = Grader::Configuration.get_instance
26 26
27 27 @room_maker = options[:room_maker]
28 28 @reporter = options[:reporter]
29 29 end
30 30
31 31 # takes a submission, asks room_maker to produce grading directories,
32 32 # calls grader scripts, and asks reporter to save the result
33 33 def grade(submission)
34 34 current_dir = FileUtils.pwd
35 35
36 36 user = submission.user
37 37 problem = submission.problem
38 38
39 39 # TODO: will have to create real exception for this
40 40 if user==nil or problem == nil
41 41 @reporter.report_error(submission,"Grading error: problem with submission")
42 42 #raise "engine: user or problem is nil"
43 43 end
44 44
45 45 # TODO: this is another hack so that output only task can be judged
46 46 if submission.language!=nil
47 47 language = submission.language.name
48 48 lang_ext = submission.language.ext
49 49 else
50 50 language = 'c'
51 51 lang_ext = 'c'
52 52 end
53 53
54 54 # FIX THIS
55 55 talk 'some hack on language'
56 56 if language == 'cpp'
57 57 language = 'c++'
58 58 end
59 59
60 60 # COMMENT: should it be only source.ext?
61 61 if problem!=nil
62 62 source_name = "#{problem.name}.#{lang_ext}"
63 63 else
64 64 source_name = "source.#{lang_ext}"
65 65 end
66 66
67 67 begin
68 68 grading_dir = @room_maker.produce_grading_room(submission)
69 69 @room_maker.save_source(submission,source_name)
70 70 problem_home = @room_maker.find_problem_home(submission)
71 71
72 72 # puts "GRADING DIR: #{grading_dir}"
73 73 # puts "PROBLEM DIR: #{problem_home}"
74 74
75 75 if !FileTest.exist?(problem_home)
76 76 raise "No test data."
77 77 end
78 78
79 79 dinit = DirInit::Manager.new(problem_home)
80 80
81 81 dinit.setup do
82 82 copy_log = copy_script(problem_home)
83 83 save_copy_log(problem_home,copy_log)
84 84 end
85 85
86 86 call_judge(problem_home,language,grading_dir,source_name)
87 87
88 88 @reporter.report(submission,"#{grading_dir}/test-result")
89 89
90 90 dinit.teardown do
91 91 copy_log = load_copy_log(problem_home)
92 92 clear_copy_log(problem_home)
93 93 clear_script(copy_log,problem_home)
94 94 end
95 95
96 96 rescue RuntimeError => msg
97 97 @reporter.report_error(submission, msg)
98 98
99 99 ensure
100 100 @room_maker.clean_up(submission)
101 101 Dir.chdir(current_dir) # this is really important
102 102 end
103 103 end
104 104
105 105 protected
106 106
107 107 def talk(str)
108 108 if @config.talkative
109 109 puts str
110 110 end
111 111 end
112 112
113 113 def call_judge(problem_home,language,grading_dir,fname)
114 114 ENV['PROBLEM_HOME'] = problem_home
115 115
116 116 talk grading_dir
117 117 Dir.chdir grading_dir
118 118 cmd = "#{problem_home}/script/judge #{language} #{fname}"
119 119 talk "CMD: #{cmd}"
120 - system(cmd)
120 + system("ruby " + cmd)
121 121 end
122 122
123 123 def get_std_script_dir
124 124 GRADER_ROOT + '/std-script'
125 125 end
126 126
127 127 def copy_script(problem_home)
128 128 script_dir = "#{problem_home}/script"
129 129 std_script_dir = get_std_script_dir
130 130
131 131 raise "std-script directory not found" if !FileTest.exist?(std_script_dir)
132 132
133 133 scripts = Dir[std_script_dir + '/*']
134 134
135 135 copied = []
136 136
137 137 scripts.each do |s|
138 138 fname = File.basename(s)
139 139 next if FileTest.directory?(s)
140 140 if !FileTest.exist?("#{script_dir}/#{fname}")
141 141 copied << fname
142 142 FileUtils.cp(s, "#{script_dir}")
143 143 end
144 144 end
145 145
146 146 return copied
147 147 end
148 148
149 149 def copy_log_filename(problem_home)
150 150 return File.join(problem_home, '.scripts_copied')
151 151 end
152 152
153 153 def save_copy_log(problem_home, log)
154 154 f = File.new(copy_log_filename(problem_home),"w")
155 155 log.each do |fname|
156 156 f.write("#{fname}\n")
157 157 end
158 158 f.close
159 159 end
160 160
161 161 def load_copy_log(problem_home)
162 162 f = File.new(copy_log_filename(problem_home),"r")
163 163 log = []
164 164 f.readlines.each do |line|
165 165 log << line.strip
166 166 end
167 167 f.close
168 168 log
169 169 end
170 170
171 171 def clear_copy_log(problem_home)
172 172 File.delete(copy_log_filename(problem_home))
173 173 end
174 174
175 175 def clear_script(log,problem_home)
176 176 log.each do |s|
177 177 FileUtils.rm("#{problem_home}/script/#{s}")
178 178 end
179 179 end
180 180
181 181 def mkdir_if_does_not_exist(dirname)
182 182 Dir.mkdir(dirname) if !FileTest.exist?(dirname)
183 183 end
184 184
185 185 end
186 186
187 187 end
Show file before | Show file after | Raw diff | Download diff | Show whitespace Increase context Show commentsHide comments
@@ -1,109 +1,109
1 1 #!/usr/bin/ruby
2 2
3 3 require 'fileutils'
4 4
5 5 ##############################
6 6 #
7 7 # Standard Compile Script
8 8 #
9 9 # Supported compilers:
10 10 # gcc, g++, and fpc.
11 11 #
12 12 ##############################
13 13
14 14 def talk(msg)
15 15 if ENV['TALKATIVE']!=nil
16 16 puts str
17 17 end
18 18 if ENV['GRADER_LOGGING']!=nil
19 19 log_fname = ENV['GRADER_LOGGING']
20 20 fp = File.open(log_fname,"a")
21 21 fp.puts("run: #{Time.new.strftime("%H:%M")} #{str}")
22 22 fp.close
23 23 end
24 24 end
25 25
26 - C_COMPILER = "/usr/bin/gcc"
27 - CPLUSPLUS_COMPILER = "/usr/bin/g++"
28 - PASCAL_COMPILER = "/usr/bin/fpc"
26 + C_COMPILER = "gcc"
27 + CPLUSPLUS_COMPILER = "g++"
28 + PASCAL_COMPILER = "fpc"
29 29
30 30 C_OPTIONS = "-O2 -s -static -std=c99 -DCONTEST -lm -Wall"
31 31 CPLUSPLUS_OPTIONS = "-O2 -s -static -DCONTEST -lm -Wall"
32 32 PASCAL_OPTIONS = "-O1 -XS -dCONTEST"
33 33
34 34 # Check for the correct number of arguments. Otherwise, print usage.
35 35 if ARGV.length == 0 or ARGV.length > 4
36 36 puts "Usage: compile <language> [<source-file>] [<output-file>] [<message-file>]"
37 37 puts
38 38 puts "<source-file> is defaulted to \"source\"."
39 39 puts "<output-file> is defaulted to \"a.out\"."
40 40 puts "<message-file> is defaulted to \"compiler_message\"."
41 41 puts
42 42 exit(127)
43 43 end
44 44
45 45 PARAMS = {
46 46 :source_file => [1,'source'],
47 47 :output_file => [2,'a.out'],
48 48 :message_file => [3,'compiler_message']
49 49 }
50 50
51 51 params = {}
52 52 params[:prog_lang] = ARGV[0]
53 53 PARAMS.each_key do |param_name|
54 54 index, default = PARAMS[param_name]
55 55 if ARGV.length > index
56 56 params[param_name] = ARGV[index]
57 57 else
58 58 params[param_name] = default
59 59 end
60 60 talk "#{param_name}: #{params[param_name]}"
61 61 end
62 62
63 63 # Remove any remaining output files or message files.
64 64 if FileTest.exists? params[:output_file]
65 65 FileUtils.rm(params[:output_file])
66 66 end
67 67 if FileTest.exists? params[:message_file]
68 68 FileUtils.rm(params[:message_file])
69 69 end
70 70
71 71 # Check if the source file exists before attempt compiling.
72 72 if !FileTest.exists? params[:source_file]
73 73 talk("ERROR: The source file does not exist!")
74 74 open(params[:message_file],"w") do |f|
75 75 f.puts "ERROR: The source file did not exist."
76 76 end
77 77 exit(127)
78 78 end
79 79
80 80 # Compile.
81 81 case params[:prog_lang]
82 82
83 83 when "c"
84 84 command = "#{C_COMPILER} #{params[:source_file]} -o #{params[:output_file]} #{C_OPTIONS} 2> #{params[:message_file]}"
85 85 system(command)
86 86
87 87 when "c++"
88 88 command = "#{CPLUSPLUS_COMPILER} #{params[:source_file]} -o #{params[:output_file]} #{CPLUSPLUS_OPTIONS} 2> #{params[:message_file]}"
89 89 system(command)
90 90
91 91 when "pas"
92 92 command = "#{PASCAL_COMPILER} #{params[:source_file]} -ooutpas #{PASCAL_OPTIONS} > #{params[:message_file]}"
93 93 system(command)
94 94 FileUtils.mv("output", params[:output_file])
95 95
96 96 else
97 97 talk("ERROR: Invalid language specified!")
98 98 open(params[:message_file],"w") do |f|
99 99 f.puts "ERROR: Invalid language specified!"
100 100 end
101 101 exit(127)
102 102 end
103 103
104 104 # Report success or failure.
105 105 if FileTest.exists? params[:output_file]
106 106 talk "Compilation was successful!"
107 107 else
108 108 talk "ERROR: Something was wrong during the compilation!"
109 109 end
Show file before | Show file after | Raw diff | Download diff | Show whitespace Increase context Show commentsHide comments
@@ -1,172 +1,174
1 1 #!/usr/bin/ruby
2 2
3 3 require 'fileutils'
4 4
5 5 def log(str='')
6 6 if ENV['TALKATIVE']!=nil
7 7 puts str
8 8 end
9 9 if ENV['GRADER_LOGGING']!=nil
10 10 log_fname = ENV['GRADER_LOGGING']
11 11 fp = File.open(log_fname,"a")
12 12 fp.puts("judge: #{Time.new.strftime("%H:%M")} #{str}")
13 13 fp.close
14 14 end
15 15 end
16 16
17 17 problem_home = ENV['PROBLEM_HOME']
18 18
19 19 def execute(command, error_message="")
20 20 if not system(command)
21 21 msg = "ERROR: #{error_message}"
22 22 log msg
23 23 raise msg
24 24 end
25 25 end
26 26
27 27 def call_and_log(error_message)
28 28 begin
29 29 yield
30 30 rescue
31 31 msg = "ERROR: #{error_message}"
32 32 log msg
33 33 raise msg
34 34 end
35 35 end
36 36
37 37 def clear_and_create_empty_dir(dir)
38 38 FileUtils.rm_rf(dir, :secure => true)
39 39 call_and_log("Cannot make directory #{dir}.") { FileUtils.mkdir(dir) }
40 40 end
41 41
42 42 # ARGV[0] --- language
43 43 # ARGV[1] --- program source file
44 44 # ARGV[2] --- test result directory
45 45 # ARGV[3] --- sandbox directory
46 46
47 47 if ARGV.length < 2 || ARGV.length > 4
48 48 puts "Usage: judge <language> <program-source> [<test-result-directory>] [<sandbox-directory>]"
49 49 puts " <sandbox-directory> is defaulted to ./sandbox"
50 50 puts " <test-result-directory> is defaulted to ./test-result"
51 51 puts "WARNING: The judge script will forcefully create the (implicitly and explicitly) specified directories and remove anything inside it."
52 52 exit(127)
53 53 end
54 54
55 55 language = ARGV[0]
56 56 if language != "c" && language != "c++" && language != "pas"
57 57 log "You specified a language that is not supported: #{language}."
58 58 exit(127)
59 59 end
60 60
61 61 source_file = ARGV[1]
62 62 if File.exist?(source_file) == false
63 63 log "The source file does not exist."
64 64 exit(127)
65 65 end
66 66
67 67 log "Making test result and sandbox directories..."
68 68
69 69 current_dir = FileUtils.pwd
70 70 current_dir.strip!
71 71
72 72 if ARGV.length >= 3
73 73 test_result_dir = ARGV[2]
74 74 else
75 75 test_result_dir = "#{current_dir}/test-result"
76 76 end
77 77
78 78 log "Test result directory: #{test_result_dir}"
79 79 clear_and_create_empty_dir(test_result_dir)
80 80
81 81 if ARGV.length >= 4
82 82 sandbox_dir = ARGV[3]
83 83 else
84 84 sandbox_dir = "#{current_dir}/sandbox"
85 85 end
86 86 log "Sandbox directory: #{sandbox_dir}"
87 87 clear_and_create_empty_dir(sandbox_dir)
88 88
89 89 # Compile
90 90 log
91 91 log "Compiling..."
92 92 call_and_log("Cannot copy the source file to #{sandbox_dir}") {
93 93 FileUtils.cp(source_file, sandbox_dir)
94 94 }
95 95 begin
96 96 Dir.chdir sandbox_dir
97 97 rescue
98 98 log "ERROR: Cannot change directory to #{sandbox_dir}."
99 99 exit(127)
100 100 end
101 - execute("#{problem_home}/script/compile #{language} #{source_file}", "Compilation error!")
102 - compile_message = open("compiler_message").read
101 + execute("ruby #{problem_home}/script/compile #{language} #{source_file}", "Compilation error!")
102 + open("compiler_message") do |f|
103 + compile_message = f.read
103 104 compile_message.strip!
105 + end
104 106 call_and_log("Cannot move the compiler message to #{test_result_dir}.") {
105 107 FileUtils.mv("compiler_message", test_result_dir)
106 108 }
107 109 if !FileTest.exist?("a.out")
108 110 log "Cannot compile the source code. See message in #{test_result_dir}/compile_message"
109 111 exit(127)
110 112 else
111 113 call_and_log("Cannot move the compiled program to #{test_result_dir}") {
112 114 FileUtils.mv("a.out",test_result_dir)
113 115 }
114 116 FileUtils.rm_rf("#{sandbox_dir}/.")
115 117 end
116 118
117 119 require "#{problem_home}/script/test_dsl.rb"
118 120 load "#{problem_home}/test_cases/all_tests.cfg"
119 121 problem = Problem.get_instance
120 122
121 123 if problem.well_formed? == false
122 124 log "The problem specification is not well formed."
123 125 exit(127)
124 126 end
125 127
126 128 # Doing the testing.
127 129 (1..(problem.num_tests)).each do |test_num|
128 130
129 131 $stdout.print "[#{test_num}]"
130 132 $stdout.flush
131 133
132 134 log "Test number: #{test_num}"
133 135 call_and_log("Cannot copy the compiled program into #{sandbox_dir}") {
134 136 FileUtils.cp("#{test_result_dir}/a.out", sandbox_dir)
135 137 }
136 138 begin
137 - execute("#{problem_home}/script/run #{language} #{test_num}", "Error occured during execution of the run script")
139 + execute("ruby #{problem_home}/script/run #{language} #{test_num}", "Error occured during execution of the run script")
138 140 rescue
139 141 # do nothing
140 142 end
141 143 call_and_log("Cannot create directory #{test_result_dir}/#{test_num}") {
142 144 FileUtils.mkdir "#{test_result_dir}/#{test_num}"
143 145 }
144 146 call_and_log("Cannot copy the result file into #{test_result_dir}/#{test_num}") {
145 147 FileUtils.mv "#{sandbox_dir}/result", "#{test_result_dir}/#{test_num}"
146 148 }
147 149 call_and_log("Cannot copy the comment file into #{test_result_dir}/#{test_num}") {
148 150 FileUtils.mv "#{sandbox_dir}/comment", "#{test_result_dir}/#{test_num}"
149 151 }
150 152 call_and_log("Cannot copy the output file into #{test_result_dir}/#{test_num}") {
151 153 FileUtils.mv "#{sandbox_dir}/output.txt", "#{test_result_dir}/#{test_num}"
152 154 }
153 155 call_and_log("Cannot clear #{sandbox_dir}") {
154 156 FileUtils.rm_rf(Dir.glob("#{sandbox_dir}/*"), :secure => true)
155 157 }
156 158 end
157 159
158 160 $stdout.print "[done]\n"
159 161
160 162 # Grade
161 163 log
162 164 log "Grading..."
163 165 begin
164 166 Dir.chdir test_result_dir
165 167 rescue
166 168 log "ERROR: Cannot change directory to #{test_result_dir}."
167 169 exit(127)
168 170 end
169 - execute("#{problem_home}/script/grade", "An error occured during grading!")
171 + execute("ruby #{problem_home}/script/grade", "An error occured during grading!")
170 172
171 173 log
172 174 log "All done!"
Show file before | Show file after | Raw diff | Download diff | Show whitespace Increase context Show commentsHide comments
@@ -1,157 +1,157
1 1 #!/usr/bin/ruby
2 2
3 3 require 'fileutils'
4 4
5 5 def log(str='')
6 6 if ENV['TALKATIVE']!=nil
7 7 puts str
8 8 end
9 9 if ENV['GRADER_LOGGING']!=nil
10 10 log_fname = ENV['GRADER_LOGGING']
11 11 fp = File.open(log_fname,"a")
12 12 fp.puts("run: #{Time.new.strftime("%H:%M")} #{str}")
13 13 fp.close
14 14 end
15 15 end
16 16
17 17 def extract_time(t)
18 18 # puts "TIME: #{t}"
19 19 if (result=/^(.*)r(.*)u(.*)s/.match(t))
20 20 {:real => result[1], :user => result[2], :sys => result[3]}
21 21 else
22 22 #{:real => 0, :user => 0, :sys => 0}
23 23 #puts "ERROR READING RUNNING TIME: #{t}"
24 24 raise "Error reading running time: #{t}"
25 25 end
26 26 end
27 27
28 28 def compile_box(source,bin)
29 29 system("g++ #{source} -o #{bin}")
30 30 end
31 31
32 32 if ARGV.length < 2 || ARGV.length > 3
33 33 puts "Usage: run <language> <test-num> [<program-name>]"
34 34 exit(127)
35 35 end
36 36
37 37 language = ARGV[0]
38 38 test_num = ARGV[1].to_i
39 39 if ARGV.length > 2
40 40 program_name = ARGV[2]
41 41 else
42 42 program_name = "a.out"
43 43 end
44 44
45 45 problem_home = ENV['PROBLEM_HOME']
46 46 require "#{problem_home}/script/test_dsl.rb"
47 47 load "#{problem_home}/test_cases/all_tests.cfg"
48 48 problem = Problem.get_instance
49 49
50 50 if problem.well_formed? == false
51 51 log "The problem specification is not well formed."
52 52 exit(127)
53 53 end
54 54
55 55 # Check if the test number is okay.
56 56 if test_num <= 0 || test_num > problem.num_tests
57 57 log "You have specified a wrong test number."
58 58 exit(127)
59 59 end
60 60
61 61 #####################################
62 62 # Set the relavant file names here. #
63 63 #####################################
64 64
65 65 input_file_name = "#{problem_home}/test_cases/#{test_num}/input-#{test_num}.txt"
66 66
67 67 #####################################
68 68
69 69 time_limit = problem.get_time_limit test_num
70 70 mem_limit = problem.get_mem_limit(test_num) * 1024
71 71
72 72 # Copy the input file.
73 73 #`cp #{problem_home}/test_cases/#{test_num}/#{input_file_name} .`
74 74
75 75 # check if box is there, if not, compile it!
76 - if !File.exists?("#{problem_home}/script/box")
76 + if !File.exists?("#{problem_home}/script/box") and !File.exists?("#{problem_home}/script/box.exe")
77 77 log "WARNING: Compiling box: to increase efficiency, it should be compile manually"
78 78 compile_box("#{problem_home}/script/box.cc",
79 79 "#{problem_home}/script/box")
80 80 end
81 81
82 82 # Hide PROBLEM_HOME
83 83 ENV['PROBLEM_HOME'] = nil
84 84
85 85 # Run the program.
86 86 #run_command = "/usr/bin/time -f \"#{time_output_format}\" 2>run_result #{problem_home}/script/box_new -a 2 -f -t #{time_limit} -m #{mem_limit} -i #{input_file_name} -o output.txt #{program_name}"
87 87 run_command = "#{problem_home}/script/box -a 2 -f -T -t #{time_limit} -m #{mem_limit} -i #{input_file_name} -o output.txt #{program_name} 2>run_result"
88 88 log "Running test #{test_num}..."
89 89 log run_command
90 90 log
91 91 system(run_command)
92 92
93 93 # Restore PROBLEM_HOME
94 94 ENV['PROBLEM_HOME'] = problem_home
95 95
96 96 # Create the result file.
97 97 result_file = File.new("result", "w")
98 98 comment_file = File.new("comment", "w")
99 99
100 100 # Check if the program actually produced any output.
101 101 run_result_file = File.new("run_result", "r")
102 102 run_result = run_result_file.readlines
103 103 run_result_file.close
104 104
105 105 run_stat = run_result[run_result.length-1]
106 106 running_time = extract_time(run_stat)
107 107
108 108 report = lambda{ |status, points, comment|
109 109 result_file.write status.strip
110 110 result_file.write "\n"
111 111 result_file.write points.to_s.strip
112 112 result_file.write "\n"
113 113 result_file.write run_stat.strip
114 114 result_file.write "\n"
115 115 result_file.close
116 116 FileUtils.rm "run_result"
117 117 # `rm output.txt` --- keep the output
118 118
119 119 comment_file.write comment
120 120
121 121 # added for debuggin --- jittat
122 122 comment_file.write "--run-result--\n"
123 123 run_result.each do |l|
124 124 comment_file.write l
125 125 end
126 126
127 127 comment_file.close
128 128
129 129 log "Done!"
130 130 exit(0)
131 131 }
132 132
133 133 if run_result[0][0,2] != "OK"
134 134 log "There was a runtime error."
135 135 report.call(run_result[0], 0, "No comment.\n")
136 136 end
137 137
138 138 if running_time[:user].to_f + running_time[:sys].to_f > time_limit
139 139 log "Time limit exceeded."
140 140 report.call("Time limit exceeded", 0, "No comment.\n")
141 141 end
142 142
143 143 # Run 'check' to evaluate the output.
144 144 #puts "There was no runtime error. Proceed to checking the output."
145 145 check_command = "#{problem_home}/script/check #{language} #{test_num}"
146 146 log "Checking the output..."
147 147 log check_command
148 - if not system(check_command)
148 + if not system("ruby " + check_command)
149 149 log "Problem with check script"
150 150 report.call("Incorrect",0,"Check script error.\n")
151 151 exit(127)
152 152 end
153 153
154 154 check_file = File.new("check_result", "r")
155 155 check_file_lines = check_file.readlines
156 156
157 157 report.call(check_file_lines[0], check_file_lines[1], "No comment.\n")
You need to be logged in to leave comments. Login now