cafe-grader-judge-scripts

Commit r257:98303143aafa public

parent | child
Description:
more commenting on script
Commit status:
[Not Reviewed]
References:
default
Diff options:
Raw Diff | Patch Diff | Download Diff | Ignore whitespace | Increase context
Comments:
0 Commit comments 0 Inline Comments
Unresolved TODOs:
There are no unresolved TODOs
Side by Side Unified
Expand All Files Collapse All Files Wide Mode Diff
Add another comment
Browse Files

r257:98303143aafa - - 3 files changed: 24 inserted, 2 deleted

Show file before | Show file after | Raw diff | Download diff | Ignore whitespace Increase context Show commentsHide comments
@@ -33,96 +33,97
33 33 def grade(submission)
34 34 current_dir = FileUtils.pwd
35 35
36 36 user = submission.user
37 37 problem = submission.problem
38 38
39 39 begin
40 40 # TODO: will have to create real exception for this
41 41 if user==nil or problem == nil
42 42 @reporter.report_error(submission,"Grading error: problem with submission")
43 43 raise "engine: user or problem is nil"
44 44 end
45 45
46 46 # TODO: this is another hack so that output only task can be judged
47 47 if submission.language!=nil
48 48 language = submission.language.name
49 49 lang_ext = submission.language.ext
50 50 else
51 51 language = 'c'
52 52 lang_ext = 'c'
53 53 end
54 54
55 55 # This is needed because older version of std-scripts/compile
56 56 # only look for c++.
57 57 if language == 'cpp'
58 58 language = 'c++'
59 59 end
60 60
61 61 # COMMENT: should it be only source.ext?
62 62 if problem!=nil
63 63 source_name = "#{problem.name}.#{lang_ext}"
64 64 else
65 65 source_name = "source.#{lang_ext}"
66 66 end
67 67
68 68 grading_dir = @room_maker.produce_grading_room(submission)
69 69 @room_maker.save_source(submission,source_name)
70 70 problem_home = @room_maker.find_problem_home(submission)
71 71
72 72 # puts "GRADING DIR: #{grading_dir}"
73 73 # puts "PROBLEM DIR: #{problem_home}"
74 74
75 75 if !FileTest.exist?(problem_home)
76 76 puts "PROBLEM DIR: #{problem_home}"
77 77 raise "engine: No test data."
78 78 end
79 79
80 80 talk "ENGINE: grading dir at #{grading_dir} is created"
81 + talk "ENGINE: located problem home at #{problem_home} is created"
81 82
82 83 # copy the source script, using lock
83 84 dinit = DirInit::Manager.new(problem_home)
84 85
85 86 # lock the directory and copy the scripts
86 87 dinit.setup do
87 88 copy_log = copy_script(problem_home)
88 89 save_copy_log(problem_home,copy_log)
89 90 talk "ENGINE: following std script is copied: #{copy_log.join ' '}"
90 91 end
91 92
92 93
93 94 call_judge(problem_home,language,grading_dir,source_name)
94 95
95 96 @reporter.report(submission,"#{grading_dir}/test-result")
96 97
97 98 # unlock the directory
98 99 dinit.teardown do
99 100 copy_log = load_copy_log(problem_home)
100 101 clear_copy_log(problem_home)
101 102 clear_script(copy_log,problem_home)
102 103 end
103 104
104 105 rescue RuntimeError => msg
105 106 @reporter.report_error(submission, msg)
106 107 puts "ERROR: #{msg}"
107 108
108 109 ensure
109 110 @room_maker.clean_up(submission)
110 111 Dir.chdir(current_dir) # this is really important
111 112 end
112 113 end
113 114
114 115 protected
115 116
116 117 def talk(str)
117 118 if @config.talkative
118 119 puts str
119 120 end
120 121 end
121 122
122 123 #change directory to problem_home
123 124 #call the "judge" script
124 125 def call_judge(problem_home,language,grading_dir,fname)
125 126 ENV['PROBLEM_HOME'] = problem_home
126 127 ENV['RUBYOPT'] = ''
127 128
128 129 Dir.chdir grading_dir
Show file before | Show file after | Raw diff | Download diff | Ignore whitespace Increase context Show commentsHide comments
@@ -113,80 +113,82
113 113 else
114 114 call_and_log("Cannot move the compiled program to #{test_result_dir}") {
115 115 FileUtils.mv("a.out",test_result_dir)
116 116 if language == "java" then Dir["*.class"].each { |file| FileUtils.mv(file,test_result_dir)} end
117 117 if language == "python" then Dir["*.pyc"].each { |file| FileUtils.mv(file,test_result_dir)} end
118 118 }
119 119 FileUtils.rm_rf("#{sandbox_dir}/.")
120 120 end
121 121
122 122
123 123 #-----------------------------------------------
124 124 # run
125 125 #-----------------------------------------------
126 126 require "#{problem_home}/script/test_dsl.rb"
127 127 load "#{problem_home}/test_cases/all_tests.cfg"
128 128 problem = Problem.get_instance
129 129
130 130 if problem.well_formed? == false
131 131 log "The problem specification is not well formed."
132 132 exit(127)
133 133 end
134 134
135 135 # Doing the testing.
136 136 log
137 137 log "JUDGE: Running each test case..."
138 138 (1..(problem.num_tests)).each do |test_num|
139 139
140 140 $stdout.print "[#{test_num}]"
141 141 $stdout.flush
142 142
143 143 call_and_log("Cannot copy the compiled program into #{sandbox_dir}") {
144 144 FileUtils.cp("#{test_result_dir}/a.out", sandbox_dir, :preserve => true)
145 145 if language == "java" then Dir["#{test_result_dir}/*.class"].each { |file| FileUtils.cp(file,sandbox_dir)} end
146 146 if language == "python" then Dir["#{test_result_dir}/*.pyc"].each { |file| FileUtils.cp(file,sandbox_dir)} end
147 147 }
148 148
149 149 #additionally copy any extra .txt file
150 150 data_files = Dir[problem_home + '/*.txt']
151 151 data_files.each do |file|
152 152 FileUtils.cp(file,sandbox_dir)
153 153 end
154 154
155 155 begin
156 156 execute("#{problem_home}/script/run #{language} #{test_num} ", "Error occured during execution of the run script")
157 157 rescue
158 158 # do nothing
159 159 end
160 160
161 +
162 + #copy the output of run script to each test-result folder
161 163 call_and_log("Cannot create directory #{test_result_dir}/#{test_num}") {
162 164 FileUtils.mkdir "#{test_result_dir}/#{test_num}"
163 165 }
164 166 call_and_log("Cannot copy the result file into #{test_result_dir}/#{test_num}") {
165 167 FileUtils.mv "#{sandbox_dir}/result", "#{test_result_dir}/#{test_num}"
166 168 }
167 169 call_and_log("Cannot copy the comment file into #{test_result_dir}/#{test_num}") {
168 170 FileUtils.mv "#{sandbox_dir}/comment", "#{test_result_dir}/#{test_num}"
169 171 }
170 172 call_and_log("Cannot copy the output file into #{test_result_dir}/#{test_num}") {
171 173 FileUtils.mv "#{sandbox_dir}/output.txt", "#{test_result_dir}/#{test_num}"
172 174 }
173 175 call_and_log("Cannot clear #{sandbox_dir}") {
174 176 FileUtils.rm_rf(Dir.glob("#{sandbox_dir}/*"), :secure => true)
175 177 }
176 178 end
177 179
178 180 $stdout.print "[done]\n"
179 181
180 182 # Grade
181 183 log
182 184 log "JUDGE: Grading..."
183 185 begin
184 186 Dir.chdir test_result_dir
185 187 rescue
186 188 log "ERROR: Cannot change directory to #{test_result_dir}."
187 189 exit(127)
188 190 end
189 191 execute("#{problem_home}/script/grade", "An error occured during grading!")
190 192
191 193 log
192 194 log "All done!"
Show file before | Show file after | Raw diff | Download diff | Ignore whitespace Increase context Show commentsHide comments
@@ -1,108 +1,127
1 1 #!/usr/bin/env ruby
2 2
3 + ##
4 + # This program should be run in the sandbox dir containing the compiled file
5 + # (or source file for script language). It will call the sandbox program with
6 + # the given input and process the output of the sandbox
7 + #
8 + # If sandbox exit normally, this program will call the "check" script to do
9 + # scoring. Otherwise, it would record the error accordingly
10 + #
11 + # This program produces several file
12 + # * result - the result from check script
13 + # * comment - comment from sandbox
14 + # * output - output of the program
15 + #
16 +
3 17 require 'fileutils'
4 18
5 19 def log(str='')
6 20 if ENV['TALKATIVE']!=nil
7 21 puts str
8 22 end
9 23 if ENV['GRADER_LOGGING']!=nil
10 24 log_fname = ENV['GRADER_LOGGING']
11 25 fp = File.open(log_fname,"a")
12 26 fp.puts("run: #{Time.new.strftime("%H:%M")} #{str}")
13 27 fp.close
14 28 end
15 29 end
16 30
17 31 def extract_time(t)
18 32 # puts "TIME: #{t}"
19 33 if (result=/^(.*)r(.*)u(.*)s/.match(t))
20 34 {:real => result[1], :user => result[2], :sys => result[3]}
21 35 else
22 36 #{:real => 0, :user => 0, :sys => 0}
23 37 #puts "ERROR READING RUNNING TIME: #{t}"
24 38 raise "Error reading running time: #{t}"
25 39 end
26 40 end
27 41
28 42 def compile_box(source,bin)
29 43 system("g++ #{source} -o #{bin}")
30 44 end
31 45
46 + #------------------------------------------
47 + # MAIN
48 + #------------------------------------------
49 +
50 + #parse parameter
32 51 if ARGV.length < 2 || ARGV.length > 3
33 52 puts "Usage: run <language> <test-num> [<program-name>]"
34 53 exit(127)
35 54 end
36 55
37 56 language = ARGV[0]
38 57 test_num = ARGV[1].to_i
39 58 if ARGV.length > 2
40 59 program_name = ARGV[2]
41 60 else
42 61 program_name = "a.out"
43 62 end
44 63
45 64 problem_home = ENV['PROBLEM_HOME']
46 65 source_name = ENV['SOURCE_NAME']
47 66 require "#{problem_home}/script/test_dsl.rb"
48 67 load "#{problem_home}/test_cases/all_tests.cfg"
49 68 problem = Problem.get_instance
50 69
51 70 sandbox_dir = Dir.getwd
52 71
53 72 if problem.well_formed? == false
54 - log "RUN: The problem specification is not well formed."
73 + log "RUN: ERROR: The problem specification is not well formed."
55 74 exit(127)
56 75 end
57 76
58 77 # Check if the test number is okay.
59 78 if test_num <= 0 || test_num > problem.num_tests
60 - log "RUN: You have specified a wrong test number."
79 + log "RUN: ERROR: You have specified a wrong test number."
61 80 exit(127)
62 81 end
63 82
64 83 #####################################
65 84 # Set the relavant file names here. #
66 85 #####################################
67 86
68 87 input_file_name = "#{problem_home}/test_cases/#{test_num}/input-#{test_num}.txt"
69 88
70 89 #####################################
71 90
72 91 time_limit = problem.get_time_limit test_num
73 92 mem_limit = problem.get_mem_limit(test_num) * 1024
74 93
75 94 # Copy the input file.
76 95 #`cp #{problem_home}/test_cases/#{test_num}/#{input_file_name} .`
77 96
78 97 # check if box is there, if not, compile it!
79 98 if !File.exists?("#{problem_home}/script/box")
80 99 log "WARNING: Compiling box: to increase efficiency, it should be compile manually"
81 100 compile_box("#{problem_home}/script/box.cc",
82 101 "#{problem_home}/script/box")
83 102 end
84 103
85 104 # Hide PROBLEM_HOME
86 105 ENV['PROBLEM_HOME'] = nil
87 106 ENV['SOURCE_NAME'] = nil
88 107
89 108 # Run the program.
90 109 #run_command = "/usr/bin/time -f \"#{time_output_format}\" 2>run_result #{problem_home}/script/box_new -a 2 -f -t #{time_limit} -m #{mem_limit} -i #{input_file_name} -o output.txt #{program_name}"
91 110 #
92 111
93 112 JAVA_OPTION = "-s set_robust_list -s futex -s clone -s getppid -s clone -s wait4 -p /usr/bin/ -p ./"
94 113 RUBY_OPTION = "-p /usr/lib64/ -p /usr/local/lib64/ -p /usr/local/lib/ -p /lib64/ -p /dev/urandom -p #{sandbox_dir}/#{program_name} -p #{sandbox_dir}/ -s set_robust_list -s sched_getaffinity -s clock_gettime -s sigaltstack -s pipe2 -s clone -s futex -s openat -s pipe -s getrandom"
95 114 PYTHON_OPTION = "-p /usr/lib64/ -p /usr/local/lib64/ -p /usr/local/lib/ -p /usr/bin/ -p /lib64/ -p /dev/urandom -p /usr/ -p #{sandbox_dir}/#{program_name} -p ./#{program_name} -p #{sandbox_dir}/#{source_name} -p /proc/sys/crypto/fips_enabled -p /proc/self/status -p /proc/mounts -p /var/lib/dpkg/status -s statfs -s set_robust_list -s openat -s sysinfo -s recvmsg -s connect -s socket -s sendto -s futex -s sigaltstack -s getrandom -E PYTHONNOUSERSITE=yes"
96 115 PHP_OPTION = "-p /usr/lib64/ -p/lib64/ -p /usr/bin/ -p #{sandbox_dir}/#{program_name} -p ./#{program_name} -p /usr/share/ -s setfsuid -s setfsgid -s openat -s set_robust_list -s futex -s clone -s socket -s connect"
97 116 HASKELL_OPTION = "-s set_robust_list -s clock_gettime -s sysinfo -s timer_create -s timer_settime -s futex -s timer_delete"
98 117
99 118 case language
100 119 when "java"
101 120 # for java, extract the classname
102 121 # wne have to add additional systemcall and we don't check the mem limit (dunno how to fix...)
103 122 classname = 'DUMMY'
104 123 File.open(program_name,"r").each do |line|
105 124 classname = line
106 125 end
107 126 #for java, we cannot really check the memory limit...
108 127 run_command = "#{problem_home}/script/box -a 3 -f -T -t #{time_limit} #{JAVA_OPTION} -i #{input_file_name} -o output.txt /usr/bin/java -A -Xmx#{mem_limit}k -A #{classname} "
You need to be logged in to leave comments. Login now