cafe-grader-judge-scripts

Commit r257:98303143aafa public

parent | child
Description:
more commenting on script
Commit status:
[Not Reviewed]
References:
default
Diff options:
Raw Diff | Patch Diff | Download Diff | Ignore whitespace | Increase context
Comments:
0 Commit comments 0 Inline Comments
Unresolved TODOs:
There are no unresolved TODOs
Side by Side Unified
Expand All Files Collapse All Files Wide Mode Diff
Add another comment
Browse Files

r257:98303143aafa - - 3 files changed: 24 inserted, 2 deleted

Show file before | Show file after | Raw diff | Download diff | Ignore whitespace Increase context Show commentsHide comments
@@ -1,176 +1,177
1 1 require 'fileutils'
2 2 require File.join(File.dirname(__FILE__),'dir_init')
3 3
4 4 module Grader
5 5
6 6 #
7 7 # A grader engine grades a submission, against anything: a test
8 8 # data, or a user submitted test data. It uses two helpers objects:
9 9 # room_maker and reporter.
10 10 #
11 11 class Engine
12 12
13 13 attr_writer :room_maker
14 14 attr_writer :reporter
15 15
16 16 def initialize(options={})
17 17 # default options
18 18 if not options.include? :room_maker
19 19 options[:room_maker] = Grader::SubmissionRoomMaker.new
20 20 end
21 21 if not options.include? :reporter
22 22 options[:reporter] = Grader::SubmissionReporter.new
23 23 end
24 24
25 25 @config = Grader::Configuration.get_instance
26 26
27 27 @room_maker = options[:room_maker]
28 28 @reporter = options[:reporter]
29 29 end
30 30
31 31 # takes a submission, asks room_maker to produce grading directories,
32 32 # calls grader scripts, and asks reporter to save the result
33 33 def grade(submission)
34 34 current_dir = FileUtils.pwd
35 35
36 36 user = submission.user
37 37 problem = submission.problem
38 38
39 39 begin
40 40 # TODO: will have to create real exception for this
41 41 if user==nil or problem == nil
42 42 @reporter.report_error(submission,"Grading error: problem with submission")
43 43 raise "engine: user or problem is nil"
44 44 end
45 45
46 46 # TODO: this is another hack so that output only task can be judged
47 47 if submission.language!=nil
48 48 language = submission.language.name
49 49 lang_ext = submission.language.ext
50 50 else
51 51 language = 'c'
52 52 lang_ext = 'c'
53 53 end
54 54
55 55 # This is needed because older version of std-scripts/compile
56 56 # only look for c++.
57 57 if language == 'cpp'
58 58 language = 'c++'
59 59 end
60 60
61 61 # COMMENT: should it be only source.ext?
62 62 if problem!=nil
63 63 source_name = "#{problem.name}.#{lang_ext}"
64 64 else
65 65 source_name = "source.#{lang_ext}"
66 66 end
67 67
68 68 grading_dir = @room_maker.produce_grading_room(submission)
69 69 @room_maker.save_source(submission,source_name)
70 70 problem_home = @room_maker.find_problem_home(submission)
71 71
72 72 # puts "GRADING DIR: #{grading_dir}"
73 73 # puts "PROBLEM DIR: #{problem_home}"
74 74
75 75 if !FileTest.exist?(problem_home)
76 76 puts "PROBLEM DIR: #{problem_home}"
77 77 raise "engine: No test data."
78 78 end
79 79
80 80 talk "ENGINE: grading dir at #{grading_dir} is created"
81 + talk "ENGINE: located problem home at #{problem_home} is created"
81 82
82 83 # copy the source script, using lock
83 84 dinit = DirInit::Manager.new(problem_home)
84 85
85 86 # lock the directory and copy the scripts
86 87 dinit.setup do
87 88 copy_log = copy_script(problem_home)
88 89 save_copy_log(problem_home,copy_log)
89 90 talk "ENGINE: following std script is copied: #{copy_log.join ' '}"
90 91 end
91 92
92 93
93 94 call_judge(problem_home,language,grading_dir,source_name)
94 95
95 96 @reporter.report(submission,"#{grading_dir}/test-result")
96 97
97 98 # unlock the directory
98 99 dinit.teardown do
99 100 copy_log = load_copy_log(problem_home)
100 101 clear_copy_log(problem_home)
101 102 clear_script(copy_log,problem_home)
102 103 end
103 104
104 105 rescue RuntimeError => msg
105 106 @reporter.report_error(submission, msg)
106 107 puts "ERROR: #{msg}"
107 108
108 109 ensure
109 110 @room_maker.clean_up(submission)
110 111 Dir.chdir(current_dir) # this is really important
111 112 end
112 113 end
113 114
114 115 protected
115 116
116 117 def talk(str)
117 118 if @config.talkative
118 119 puts str
119 120 end
120 121 end
121 122
122 123 #change directory to problem_home
123 124 #call the "judge" script
124 125 def call_judge(problem_home,language,grading_dir,fname)
125 126 ENV['PROBLEM_HOME'] = problem_home
126 127 ENV['RUBYOPT'] = ''
127 128
128 129 Dir.chdir grading_dir
129 130 script_name = "#{problem_home}/script/judge"
130 131 cmd = "#{script_name} #{language} #{fname}"
131 132 talk "ENGINE: Calling Judge at #{cmd}"
132 133 warn "ERROR: file does not exists #{script_name}" unless File.exists? script_name
133 134 system(cmd)
134 135 end
135 136
136 137 def get_std_script_dir
137 138 GRADER_ROOT + '/std-script'
138 139 end
139 140
140 141 #copy any script presented in std-script directory that is not in the problem_home
141 142 #this allow a problem setter to provide their own version for each script
142 143 #in case that they want to hack something
143 144 def copy_script(problem_home)
144 145 script_dir = "#{problem_home}/script"
145 146 std_script_dir = get_std_script_dir
146 147
147 148 raise "engine: std-script directory not found" if !FileTest.exist?(std_script_dir)
148 149
149 150 scripts = Dir[std_script_dir + '/*']
150 151
151 152 copied = []
152 153
153 154 scripts.each do |s|
154 155 fname = File.basename(s)
155 156 next if FileTest.directory?(s)
156 157 if !FileTest.exist?("#{script_dir}/#{fname}")
157 158 copied << fname
158 159 FileUtils.cp(s, "#{script_dir}", :preserve => true)
159 160 end
160 161 end
161 162
162 163 return copied
163 164 end
164 165
165 166 def copy_log_filename(problem_home)
166 167 return File.join(problem_home, '.scripts_copied')
167 168 end
168 169
169 170 def save_copy_log(problem_home, log)
170 171 f = File.new(copy_log_filename(problem_home),"w")
171 172 log.each do |fname|
172 173 f.write("#{fname}\n")
173 174 end
174 175 f.close
175 176 end
176 177
Show file before | Show file after | Raw diff | Download diff | Ignore whitespace Increase context Show commentsHide comments
@@ -65,128 +65,130
65 65 exit(127)
66 66 end
67 67
68 68 log "JUDGE: Making test result and sandbox directories..."
69 69
70 70 current_dir = FileUtils.pwd
71 71 current_dir.strip!
72 72
73 73 if ARGV.length >= 3
74 74 test_result_dir = ARGV[2]
75 75 else
76 76 test_result_dir = "#{current_dir}/test-result"
77 77 end
78 78
79 79 log "JUDGE: Test result directory: #{test_result_dir}"
80 80 clear_and_create_empty_dir(test_result_dir)
81 81
82 82 if ARGV.length >= 4
83 83 sandbox_dir = ARGV[3]
84 84 else
85 85 sandbox_dir = "#{current_dir}/sandbox"
86 86 end
87 87 log "JUDGE: Sandbox directory: #{sandbox_dir}"
88 88 clear_and_create_empty_dir(sandbox_dir)
89 89
90 90 # ------------------------------
91 91 # Compile
92 92 # ------------------------------
93 93 log "JUDGE: Compiling..."
94 94 log
95 95 call_and_log("Cannot copy the source file to #{sandbox_dir}") {
96 96 FileUtils.cp(source_file, sandbox_dir)
97 97 }
98 98 begin
99 99 Dir.chdir sandbox_dir
100 100 rescue
101 101 log "JUDGE: ERROR: Cannot change directory to #{sandbox_dir}."
102 102 exit(127)
103 103 end
104 104 execute("#{problem_home}/script/compile #{language} #{source_file}", "Compilation error!")
105 105 compile_message = open("compiler_message").read
106 106 compile_message.strip!
107 107 call_and_log("Cannot move the compiler message to #{test_result_dir}.") {
108 108 FileUtils.mv("compiler_message", test_result_dir)
109 109 }
110 110 if !FileTest.exist?("a.out")
111 111 log "JUDGE: EROOR: Cannot compile the source code. See message in #{test_result_dir}/compile_message"
112 112 exit(127)
113 113 else
114 114 call_and_log("Cannot move the compiled program to #{test_result_dir}") {
115 115 FileUtils.mv("a.out",test_result_dir)
116 116 if language == "java" then Dir["*.class"].each { |file| FileUtils.mv(file,test_result_dir)} end
117 117 if language == "python" then Dir["*.pyc"].each { |file| FileUtils.mv(file,test_result_dir)} end
118 118 }
119 119 FileUtils.rm_rf("#{sandbox_dir}/.")
120 120 end
121 121
122 122
123 123 #-----------------------------------------------
124 124 # run
125 125 #-----------------------------------------------
126 126 require "#{problem_home}/script/test_dsl.rb"
127 127 load "#{problem_home}/test_cases/all_tests.cfg"
128 128 problem = Problem.get_instance
129 129
130 130 if problem.well_formed? == false
131 131 log "The problem specification is not well formed."
132 132 exit(127)
133 133 end
134 134
135 135 # Doing the testing.
136 136 log
137 137 log "JUDGE: Running each test case..."
138 138 (1..(problem.num_tests)).each do |test_num|
139 139
140 140 $stdout.print "[#{test_num}]"
141 141 $stdout.flush
142 142
143 143 call_and_log("Cannot copy the compiled program into #{sandbox_dir}") {
144 144 FileUtils.cp("#{test_result_dir}/a.out", sandbox_dir, :preserve => true)
145 145 if language == "java" then Dir["#{test_result_dir}/*.class"].each { |file| FileUtils.cp(file,sandbox_dir)} end
146 146 if language == "python" then Dir["#{test_result_dir}/*.pyc"].each { |file| FileUtils.cp(file,sandbox_dir)} end
147 147 }
148 148
149 149 #additionally copy any extra .txt file
150 150 data_files = Dir[problem_home + '/*.txt']
151 151 data_files.each do |file|
152 152 FileUtils.cp(file,sandbox_dir)
153 153 end
154 154
155 155 begin
156 156 execute("#{problem_home}/script/run #{language} #{test_num} ", "Error occured during execution of the run script")
157 157 rescue
158 158 # do nothing
159 159 end
160 160
161 +
162 + #copy the output of run script to each test-result folder
161 163 call_and_log("Cannot create directory #{test_result_dir}/#{test_num}") {
162 164 FileUtils.mkdir "#{test_result_dir}/#{test_num}"
163 165 }
164 166 call_and_log("Cannot copy the result file into #{test_result_dir}/#{test_num}") {
165 167 FileUtils.mv "#{sandbox_dir}/result", "#{test_result_dir}/#{test_num}"
166 168 }
167 169 call_and_log("Cannot copy the comment file into #{test_result_dir}/#{test_num}") {
168 170 FileUtils.mv "#{sandbox_dir}/comment", "#{test_result_dir}/#{test_num}"
169 171 }
170 172 call_and_log("Cannot copy the output file into #{test_result_dir}/#{test_num}") {
171 173 FileUtils.mv "#{sandbox_dir}/output.txt", "#{test_result_dir}/#{test_num}"
172 174 }
173 175 call_and_log("Cannot clear #{sandbox_dir}") {
174 176 FileUtils.rm_rf(Dir.glob("#{sandbox_dir}/*"), :secure => true)
175 177 }
176 178 end
177 179
178 180 $stdout.print "[done]\n"
179 181
180 182 # Grade
181 183 log
182 184 log "JUDGE: Grading..."
183 185 begin
184 186 Dir.chdir test_result_dir
185 187 rescue
186 188 log "ERROR: Cannot change directory to #{test_result_dir}."
187 189 exit(127)
188 190 end
189 191 execute("#{problem_home}/script/grade", "An error occured during grading!")
190 192
191 193 log
192 194 log "All done!"
Show file before | Show file after | Raw diff | Download diff | Ignore whitespace Increase context Show commentsHide comments
@@ -1,156 +1,175
1 1 #!/usr/bin/env ruby
2 2
3 + ##
4 + # This program should be run in the sandbox dir containing the compiled file
5 + # (or source file for script language). It will call the sandbox program with
6 + # the given input and process the output of the sandbox
7 + #
8 + # If sandbox exit normally, this program will call the "check" script to do
9 + # scoring. Otherwise, it would record the error accordingly
10 + #
11 + # This program produces several file
12 + # * result - the result from check script
13 + # * comment - comment from sandbox
14 + # * output - output of the program
15 + #
16 +
3 17 require 'fileutils'
4 18
5 19 def log(str='')
6 20 if ENV['TALKATIVE']!=nil
7 21 puts str
8 22 end
9 23 if ENV['GRADER_LOGGING']!=nil
10 24 log_fname = ENV['GRADER_LOGGING']
11 25 fp = File.open(log_fname,"a")
12 26 fp.puts("run: #{Time.new.strftime("%H:%M")} #{str}")
13 27 fp.close
14 28 end
15 29 end
16 30
17 31 def extract_time(t)
18 32 # puts "TIME: #{t}"
19 33 if (result=/^(.*)r(.*)u(.*)s/.match(t))
20 34 {:real => result[1], :user => result[2], :sys => result[3]}
21 35 else
22 36 #{:real => 0, :user => 0, :sys => 0}
23 37 #puts "ERROR READING RUNNING TIME: #{t}"
24 38 raise "Error reading running time: #{t}"
25 39 end
26 40 end
27 41
28 42 def compile_box(source,bin)
29 43 system("g++ #{source} -o #{bin}")
30 44 end
31 45
46 + #------------------------------------------
47 + # MAIN
48 + #------------------------------------------
49 +
50 + #parse parameter
32 51 if ARGV.length < 2 || ARGV.length > 3
33 52 puts "Usage: run <language> <test-num> [<program-name>]"
34 53 exit(127)
35 54 end
36 55
37 56 language = ARGV[0]
38 57 test_num = ARGV[1].to_i
39 58 if ARGV.length > 2
40 59 program_name = ARGV[2]
41 60 else
42 61 program_name = "a.out"
43 62 end
44 63
45 64 problem_home = ENV['PROBLEM_HOME']
46 65 source_name = ENV['SOURCE_NAME']
47 66 require "#{problem_home}/script/test_dsl.rb"
48 67 load "#{problem_home}/test_cases/all_tests.cfg"
49 68 problem = Problem.get_instance
50 69
51 70 sandbox_dir = Dir.getwd
52 71
53 72 if problem.well_formed? == false
54 - log "RUN: The problem specification is not well formed."
73 + log "RUN: ERROR: The problem specification is not well formed."
55 74 exit(127)
56 75 end
57 76
58 77 # Check if the test number is okay.
59 78 if test_num <= 0 || test_num > problem.num_tests
60 - log "RUN: You have specified a wrong test number."
79 + log "RUN: ERROR: You have specified a wrong test number."
61 80 exit(127)
62 81 end
63 82
64 83 #####################################
65 84 # Set the relavant file names here. #
66 85 #####################################
67 86
68 87 input_file_name = "#{problem_home}/test_cases/#{test_num}/input-#{test_num}.txt"
69 88
70 89 #####################################
71 90
72 91 time_limit = problem.get_time_limit test_num
73 92 mem_limit = problem.get_mem_limit(test_num) * 1024
74 93
75 94 # Copy the input file.
76 95 #`cp #{problem_home}/test_cases/#{test_num}/#{input_file_name} .`
77 96
78 97 # check if box is there, if not, compile it!
79 98 if !File.exists?("#{problem_home}/script/box")
80 99 log "WARNING: Compiling box: to increase efficiency, it should be compile manually"
81 100 compile_box("#{problem_home}/script/box.cc",
82 101 "#{problem_home}/script/box")
83 102 end
84 103
85 104 # Hide PROBLEM_HOME
86 105 ENV['PROBLEM_HOME'] = nil
87 106 ENV['SOURCE_NAME'] = nil
88 107
89 108 # Run the program.
90 109 #run_command = "/usr/bin/time -f \"#{time_output_format}\" 2>run_result #{problem_home}/script/box_new -a 2 -f -t #{time_limit} -m #{mem_limit} -i #{input_file_name} -o output.txt #{program_name}"
91 110 #
92 111
93 112 JAVA_OPTION = "-s set_robust_list -s futex -s clone -s getppid -s clone -s wait4 -p /usr/bin/ -p ./"
94 113 RUBY_OPTION = "-p /usr/lib64/ -p /usr/local/lib64/ -p /usr/local/lib/ -p /lib64/ -p /dev/urandom -p #{sandbox_dir}/#{program_name} -p #{sandbox_dir}/ -s set_robust_list -s sched_getaffinity -s clock_gettime -s sigaltstack -s pipe2 -s clone -s futex -s openat -s pipe -s getrandom"
95 114 PYTHON_OPTION = "-p /usr/lib64/ -p /usr/local/lib64/ -p /usr/local/lib/ -p /usr/bin/ -p /lib64/ -p /dev/urandom -p /usr/ -p #{sandbox_dir}/#{program_name} -p ./#{program_name} -p #{sandbox_dir}/#{source_name} -p /proc/sys/crypto/fips_enabled -p /proc/self/status -p /proc/mounts -p /var/lib/dpkg/status -s statfs -s set_robust_list -s openat -s sysinfo -s recvmsg -s connect -s socket -s sendto -s futex -s sigaltstack -s getrandom -E PYTHONNOUSERSITE=yes"
96 115 PHP_OPTION = "-p /usr/lib64/ -p/lib64/ -p /usr/bin/ -p #{sandbox_dir}/#{program_name} -p ./#{program_name} -p /usr/share/ -s setfsuid -s setfsgid -s openat -s set_robust_list -s futex -s clone -s socket -s connect"
97 116 HASKELL_OPTION = "-s set_robust_list -s clock_gettime -s sysinfo -s timer_create -s timer_settime -s futex -s timer_delete"
98 117
99 118 case language
100 119 when "java"
101 120 # for java, extract the classname
102 121 # wne have to add additional systemcall and we don't check the mem limit (dunno how to fix...)
103 122 classname = 'DUMMY'
104 123 File.open(program_name,"r").each do |line|
105 124 classname = line
106 125 end
107 126 #for java, we cannot really check the memory limit...
108 127 run_command = "#{problem_home}/script/box -a 3 -f -T -t #{time_limit} #{JAVA_OPTION} -i #{input_file_name} -o output.txt /usr/bin/java -A -Xmx#{mem_limit}k -A #{classname} "
109 128 when "ruby"
110 129 run_command = "#{problem_home}/script/box -a 2 -f -T -t #{time_limit*=2} -m #{mem_limit} #{RUBY_OPTION} -i #{input_file_name} -o output.txt /usr/bin/ruby #{program_name} "
111 130 when "python"
112 131 run_command = "#{problem_home}/script/box -a 2 -f -T -t #{time_limit*=2} -m #{[512 * 1024,mem_limit].max} #{PYTHON_OPTION} -i #{input_file_name} -o output.txt /usr/bin/python3 #{program_name} "
113 132 when "haskell"
114 133 run_command = "#{problem_home}/script/box -a 2 -f -T -t #{time_limit} -m #{[512 * 1024,mem_limit].max} #{HASKELL_OPTION} -i #{input_file_name} -o output.txt #{program_name} "
115 134 when "php"
116 135 run_command = "#{problem_home}/script/box -a 2 -f -T -t #{time_limit*=2} -m #{[512 * 1024,mem_limit].max} #{PHP_OPTION} -i #{input_file_name} -o output.txt /usr/bin/php -A -d -A memory_limit=#{mem_limit}k -A #{program_name} "
117 136 else # for c++, pascal, we do the normal checking
118 137 run_command = "#{problem_home}/script/box -a 2 -f -T -t #{time_limit} -m #{mem_limit} -i #{input_file_name} -o output.txt #{program_name} "
119 138 end
120 139
121 140
122 141 log "RUN: Running test #{test_num}..."
123 142 log "RUN: Run command = [#{run_command}]"
124 143 log
125 144 system(run_command,err: 'run_result')
126 145
127 146 # Restore PROBLEM_HOME
128 147 ENV['PROBLEM_HOME'] = problem_home
129 148
130 149 # Create the result file.
131 150 result_file = File.new("result", "w")
132 151 comment_file = File.new("comment", "w")
133 152
134 153 # Check if the program actually produced any output.
135 154 run_result_file = File.new("run_result", "r")
136 155 run_result = run_result_file.readlines
137 156 run_result_file.close
138 157
139 158 run_stat = run_result[run_result.length-1]
140 159 running_time = extract_time(run_stat)
141 160
142 161 report = lambda{ |status, points, comment|
143 162 result_file.write status.strip
144 163 result_file.write "\n"
145 164 result_file.write points.to_s.strip
146 165 result_file.write "\n"
147 166 result_file.write run_stat.strip
148 167 result_file.write "\n"
149 168 result_file.close
150 169 FileUtils.rm "run_result"
151 170 # `rm output.txt` --- keep the output
152 171
153 172 comment_file.write comment
154 173
155 174 # added for debuggin --- jittat
156 175 comment_file.write "--run-result--\n"
You need to be logged in to leave comments. Login now