cafe-grader-judge-scripts

Commit r161:d5a0f0a406b5 public

parent | child
Description:
clean up run command to check for running time / memory merge check.float
Commit status:
[Not Reviewed]
References:
merge algo
Diff options:
Raw Diff | Patch Diff | Download Diff | Ignore whitespace | Increase context
Comments:
0 Commit comments 0 Inline Comments
Unresolved TODOs:
There are no unresolved TODOs
Side by Side Unified
Expand All Files Collapse All Files Wide Mode Diff
Add another comment
Browse Files

r161:d5a0f0a406b5 - - 4 files changed: 82 inserted, 9 deleted

Show file before | Show file after | Raw diff | Download diff | Ignore whitespace Increase context Show commentsHide comments
@@ -0,0 +1,66
1 + #!/usr/bin/env ruby
2 +
3 + problem_home = ENV['PROBLEM_HOME']
4 + require "#{problem_home}/script/test_dsl.rb"
5 +
6 + if ARGV.length < 2
7 + puts "Usage: check <language> <test-number> [<output-file>]"
8 + exit(0)
9 + end
10 +
11 + language = ARGV[0]
12 + test_num = ARGV[1].to_i
13 + if ARGV.length >= 3
14 + output_file_name = ARGV[2]
15 + else
16 + output_file_name = "output.txt"
17 + end
18 +
19 + load "#{problem_home}/test_cases/all_tests.cfg"
20 + problem = Problem.get_instance
21 +
22 + output_file = File.new(output_file_name, "r")
23 + answer_file = File.new("#{problem_home}/test_cases/#{test_num}/answer-#{test_num}.txt")
24 + result_file = File.new("check_result", "w")
25 +
26 + output_file_content = output_file.read
27 + answer_file_content = answer_file.read
28 +
29 + report_correct = lambda {
30 + result_file.write "Correct\n"
31 + result_file.write problem.get_score(test_num)
32 + result_file.write "\n"
33 + result_file.close
34 + exit(0)
35 + }
36 +
37 + report_wrong = lambda {
38 + result_file.write "Incorrect\n"
39 + result_file.write "0\n"
40 + result_file.close
41 + exit(0)
42 + }
43 +
44 + ##################
45 + # Your code here #
46 + ##################
47 +
48 + ########### THIS IS FOR CHECKING FLOAT with EPSILON error ##########
49 +
50 + EPSILON = 0.000001
51 +
52 + out_items = output_file_content.split
53 + ans_items = answer_file_content.split
54 +
55 + if out_items.length != ans_items.length
56 + report_wrong.call
57 + else
58 + out_items.length.times do |i|
59 + out_value = out_items[i].to_f
60 + ans_value = ans_items[i].to_f
61 + if (out_value - ans_value).abs > EPSILON * [out_value.abs,ans_value.abs].max
62 + report_wrong.call
63 + end
64 + end
65 + report_correct.call
66 + end
Show file before | Show file after | Raw diff | Download diff | Ignore whitespace Increase context Show commentsHide comments
@@ -126,40 +126,41
126 126 command = "#{RUBY_INTERPRETER} -c #{params[:source_file]} 2> #{params[:message_file]}"
127 127 if system(command)
128 128 File.open(params[:output_file],"w") do |out_file|
129 129 out_file.puts "#!#{RUBY_INTERPRETER}"
130 130 File.open(params[:source_file],"r").each do |line|
131 131 out_file.print line
132 132 end
133 133 end
134 134 File.chmod(0755, params[:output_file])
135 135 end
136 136
137 137 when "python"
138 138 command = "#{PYTHON_CHECKER} #{params[:source_file]} > #{params[:message_file]}"
139 139 if system(command)
140 140 #compile to python bytecode
141 141 command = "#{PYTHON_INTERPRETER} -m py_compile #{params[:source_file]}"
142 142 puts "compile: #{command}"
143 143 system(command)
144 144 puts "pwd: " + Dir.pwd
145 145 Dir.new('.').each {|file| puts file}
146 146 File.open(params[:output_file],"w") do |out_file|
147 147 out_file.puts "#!#{PYTHON_INTERPRETER} #{params[:source_file]}c"
148 148 end
149 149 File.chmod(0755, params[:output_file])
150 + FileUtils.cp("#{params[:source_file]}c",params[:output_file])
150 151 end
151 152
152 153 else
153 154 talk("ERROR: Invalid language specified!")
154 155 open(params[:message_file],"w") do |f|
155 156 f.puts "ERROR: Invalid language specified!"
156 157 end
157 158 exit(127)
158 159 end
159 160
160 161 # Report success or failure.
161 162 if FileTest.exists? params[:output_file]
162 163 talk "Compilation was successful!"
163 164 else
164 165 talk "ERROR: Something was wrong during the compilation!"
165 166 end
Show file before | Show file after | Raw diff | Download diff | Ignore whitespace Increase context Show commentsHide comments
@@ -38,48 +38,49
38 38 FileUtils.rm_rf(dir, :secure => true)
39 39 call_and_log("Cannot make directory #{dir}.") { FileUtils.mkdir(dir) }
40 40 end
41 41
42 42 # ARGV[0] --- language
43 43 # ARGV[1] --- program source file
44 44 # ARGV[2] --- test result directory
45 45 # ARGV[3] --- sandbox directory
46 46
47 47 if ARGV.length < 2 || ARGV.length > 4
48 48 puts "Usage: judge <language> <program-source> [<test-result-directory>] [<sandbox-directory>]"
49 49 puts " <sandbox-directory> is defaulted to ./sandbox"
50 50 puts " <test-result-directory> is defaulted to ./test-result"
51 51 puts "WARNING: The judge script will forcefully create the (implicitly and explicitly) specified directories and remove anything inside it."
52 52 exit(127)
53 53 end
54 54
55 55 language = ARGV[0]
56 56 if language != "c" && language != "c++" && language != "pas" && language != "java" && language != "ruby" && language != "python"
57 57 log "You specified a language that is not supported: #{language}."
58 58 exit(127)
59 59 end
60 60
61 61 source_file = ARGV[1]
62 + ENV['SOURCE_NAME'] = source_file
62 63 if File.exist?(source_file) == false
63 64 log "The source file does not exist."
64 65 exit(127)
65 66 end
66 67
67 68 log "Making test result and sandbox directories..."
68 69
69 70 current_dir = FileUtils.pwd
70 71 current_dir.strip!
71 72
72 73 if ARGV.length >= 3
73 74 test_result_dir = ARGV[2]
74 75 else
75 76 test_result_dir = "#{current_dir}/test-result"
76 77 end
77 78
78 79 log "Test result directory: #{test_result_dir}"
79 80 clear_and_create_empty_dir(test_result_dir)
80 81
81 82 if ARGV.length >= 4
82 83 sandbox_dir = ARGV[3]
83 84 else
84 85 sandbox_dir = "#{current_dir}/sandbox"
85 86 end
@@ -119,49 +120,49
119 120 require "#{problem_home}/script/test_dsl.rb"
120 121 load "#{problem_home}/test_cases/all_tests.cfg"
121 122 problem = Problem.get_instance
122 123
123 124 if problem.well_formed? == false
124 125 log "The problem specification is not well formed."
125 126 exit(127)
126 127 end
127 128
128 129 # Doing the testing.
129 130 (1..(problem.num_tests)).each do |test_num|
130 131
131 132 $stdout.print "[#{test_num}]"
132 133 $stdout.flush
133 134
134 135 log "Test number: #{test_num}"
135 136
136 137 call_and_log("Cannot copy the compiled program into #{sandbox_dir}") {
137 138 FileUtils.cp("#{test_result_dir}/a.out", sandbox_dir, :preserve => true)
138 139 if language == "java" then Dir["#{test_result_dir}/*.class"].each { |file| FileUtils.cp(file,sandbox_dir)} end
139 140 if language == "python" then Dir["#{test_result_dir}/*.pyc"].each { |file| FileUtils.cp(file,sandbox_dir)} end
140 141 }
141 142
142 143 begin
143 - execute("#{problem_home}/script/run #{language} #{test_num}", "Error occured during execution of the run script")
144 + execute("#{problem_home}/script/run #{language} #{test_num} ", "Error occured during execution of the run script")
144 145 rescue
145 146 # do nothing
146 147 end
147 148
148 149 call_and_log("Cannot create directory #{test_result_dir}/#{test_num}") {
149 150 FileUtils.mkdir "#{test_result_dir}/#{test_num}"
150 151 }
151 152 call_and_log("Cannot copy the result file into #{test_result_dir}/#{test_num}") {
152 153 FileUtils.mv "#{sandbox_dir}/result", "#{test_result_dir}/#{test_num}"
153 154 }
154 155 call_and_log("Cannot copy the comment file into #{test_result_dir}/#{test_num}") {
155 156 FileUtils.mv "#{sandbox_dir}/comment", "#{test_result_dir}/#{test_num}"
156 157 }
157 158 call_and_log("Cannot copy the output file into #{test_result_dir}/#{test_num}") {
158 159 FileUtils.mv "#{sandbox_dir}/output.txt", "#{test_result_dir}/#{test_num}"
159 160 }
160 161 call_and_log("Cannot clear #{sandbox_dir}") {
161 162 FileUtils.rm_rf(Dir.glob("#{sandbox_dir}/*"), :secure => true)
162 163 }
163 164 end
164 165
165 166 $stdout.print "[done]\n"
166 167
167 168 # Grade
Show file before | Show file after | Raw diff | Download diff | Ignore whitespace Increase context Show commentsHide comments
@@ -22,159 +22,164
22 22 #{:real => 0, :user => 0, :sys => 0}
23 23 #puts "ERROR READING RUNNING TIME: #{t}"
24 24 raise "Error reading running time: #{t}"
25 25 end
26 26 end
27 27
28 28 def compile_box(source,bin)
29 29 system("g++ #{source} -o #{bin}")
30 30 end
31 31
32 32 if ARGV.length < 2 || ARGV.length > 3
33 33 puts "Usage: run <language> <test-num> [<program-name>]"
34 34 exit(127)
35 35 end
36 36
37 37 language = ARGV[0]
38 38 test_num = ARGV[1].to_i
39 39 if ARGV.length > 2
40 40 program_name = ARGV[2]
41 41 else
42 42 program_name = "a.out"
43 43 end
44 44
45 45 problem_home = ENV['PROBLEM_HOME']
46 + source_name = ENV['SOURCE_NAME']
46 47 require "#{problem_home}/script/test_dsl.rb"
47 48 load "#{problem_home}/test_cases/all_tests.cfg"
48 49 problem = Problem.get_instance
49 50
51 + sandbox_dir = Dir.getwd
52 +
50 53 if problem.well_formed? == false
51 54 log "The problem specification is not well formed."
52 55 exit(127)
53 56 end
54 57
55 58 # Check if the test number is okay.
56 59 if test_num <= 0 || test_num > problem.num_tests
57 60 log "You have specified a wrong test number."
58 61 exit(127)
59 62 end
60 63
61 64 #####################################
62 65 # Set the relavant file names here. #
63 66 #####################################
64 67
65 68 input_file_name = "#{problem_home}/test_cases/#{test_num}/input-#{test_num}.txt"
66 69
67 70 #####################################
68 71
69 72 time_limit = problem.get_time_limit test_num
70 73 mem_limit = problem.get_mem_limit(test_num) * 1024
71 74
72 75 # Copy the input file.
73 76 #`cp #{problem_home}/test_cases/#{test_num}/#{input_file_name} .`
74 77
75 78 # check if box is there, if not, compile it!
76 79 if !File.exists?("#{problem_home}/script/box")
77 80 log "WARNING: Compiling box: to increase efficiency, it should be compile manually"
78 81 compile_box("#{problem_home}/script/box.cc",
79 82 "#{problem_home}/script/box")
80 83 end
81 84
82 85 # Hide PROBLEM_HOME
83 86 ENV['PROBLEM_HOME'] = nil
87 + ENV['SOURCE_NAME'] = nil
84 88
85 89 # Run the program.
86 90 #run_command = "/usr/bin/time -f \"#{time_output_format}\" 2>run_result #{problem_home}/script/box_new -a 2 -f -t #{time_limit} -m #{mem_limit} -i #{input_file_name} -o output.txt #{program_name}"
87 91 #
88 92
89 -
90 -
93 + JAVA_OPTION = "-s set_robust_list -s futex -s clone -s getppid -s clone -s wait4 -p /usr/bin/ -p ./"
94 + RUBY_OPTION = "-p /usr/lib64/ -p /lib64/ -p /dev/urandom -p #{sandbox_dir}/#{program_name} -s set_robust_list -s sched_getaffinity -s clock_gettime -s sigaltstack -s pipe2 -s clone -s futex -s openat"
95 + PYTHON_OPTION = "-p /usr/lib64/ -p /lib64/ -p /usr/bin/ -p /usr/local/lib64/ -p /usr/local/lib/ -p #{sandbox_dir}/#{program_name} -p ./#{program_name} -p #{sandbox_dir}/#{source_name} -s set_robust_list -s openat -s recvmsg -s connect -s socket -s sendto -E PYTHONNOUSERSITE=yes"
91 96
92 97 case language
93 98 when "java"
94 -
95 99 # for java, extract the classname
96 100 # wne have to add additional systemcall and we don't check the mem limit (dunno how to fix...)
97 101 classname = 'DUMMY'
98 102 File.open(program_name,"r").each do |line|
99 103 classname = line
100 104 end
101 - run_command = "#{problem_home}/script/box -T -t #{time_limit} -s getppid -s clone -s wait4 -p /usr/bin/ -p ./ -i #{input_file_name} -o output.txt /usr/bin/java #{classname} 2>run_result"
105 + #for java, we cannot really check the memory limit...
106 + run_command = "#{problem_home}/script/box -a 3 -f -T -t #{time_limit} #{JAVA_OPTION} -i #{input_file_name} -o output.txt /usr/bin/java #{classname} 2>run_result"
102 107 when "ruby"
103 - run_command = "#{problem_home}/script/box -T -t #{time_limit} -s getppid -s wait4 -s clone -s set_robust_list -s futex -s sigaltstack -p /dev/urandom -p ./ -p /home/dae/.rvm/rubies/ruby-1.9.2-p320/ -p #{problem_home}/ -i #{input_file_name} -o output.txt #{program_name} 2>run_result"
108 + run_command = "#{problem_home}/script/box -a 2 -f -T -t #{time_limit} -m #{mem_limit} #{RUBY_OPTION} -i #{input_file_name} -o output.txt /usr/bin/ruby #{program_name} 2>run_result"
104 109 when "python"
105 - #this code just run without any checking
106 - run_command = "#{problem_home}/script/box -T -t #{time_limit} -p #{problem_home}/ -i #{input_file_name} -o output.txt #{program_name} 2>run_result"
110 + run_command = "#{problem_home}/script/box -a 2 -f -T -t #{time_limit} -m #{mem_limit} #{PYTHON_OPTION} -i #{input_file_name} -o output.txt /usr/bin/python #{program_name} 2>run_result"
107 111 else # for c++, pascal, we do the normal checking
108 112 run_command = "#{problem_home}/script/box -a 2 -f -T -t #{time_limit} -m #{mem_limit} -i #{input_file_name} -o output.txt #{program_name} 2>run_result"
109 113 end
110 114
111 115
112 116 log "Running test #{test_num}..."
113 117 log run_command
114 - log
118 + log
115 119 system(run_command)
116 120
117 121 # Restore PROBLEM_HOME
118 122 ENV['PROBLEM_HOME'] = problem_home
119 123
120 124 # Create the result file.
121 125 result_file = File.new("result", "w")
122 126 comment_file = File.new("comment", "w")
123 127
124 128 # Check if the program actually produced any output.
125 129 run_result_file = File.new("run_result", "r")
126 130 run_result = run_result_file.readlines
127 131 run_result_file.close
128 132
129 133 run_stat = run_result[run_result.length-1]
130 134 running_time = extract_time(run_stat)
131 135
132 136 report = lambda{ |status, points, comment|
133 137 result_file.write status.strip
134 138 result_file.write "\n"
135 139 result_file.write points.to_s.strip
136 140 result_file.write "\n"
137 141 result_file.write run_stat.strip
138 142 result_file.write "\n"
139 143 result_file.close
140 144 FileUtils.rm "run_result"
141 145 # `rm output.txt` --- keep the output
142 146
143 147 comment_file.write comment
144 148
145 149 # added for debuggin --- jittat
146 150 comment_file.write "--run-result--\n"
147 151 run_result.each do |l|
148 152 comment_file.write l
149 153 end
150 154
151 155 comment_file.close
152 156
153 157 log "Done!"
154 158 exit(0)
155 159 }
156 160
161 +
157 162 if run_result[0][0,2] != "OK"
158 163 log "There was a runtime error."
159 164 report.call(run_result[0], 0, "No comment.\n")
160 165 end
161 166
162 167 if running_time[:user].to_f > time_limit
163 168 log "Time limit exceeded."
164 169 report.call("Time limit exceeded", 0, "No comment.\n")
165 170 end
166 171
167 172 # Run 'check' to evaluate the output.
168 173 #puts "There was no runtime error. Proceed to checking the output."
169 174 check_command = "#{problem_home}/script/check #{language} #{test_num}"
170 175 log "Checking the output..."
171 176 log check_command
172 177 if not system(check_command)
173 178 log "Problem with check script"
174 179 report.call("Incorrect",0,"Check script error.\n")
175 180 exit(127)
176 181 end
177 182
178 183 check_file = File.new("check_result", "r")
179 184 check_file_lines = check_file.readlines
180 185
You need to be logged in to leave comments. Login now