cafe-grader-judge-scripts Commit

cafe-grader-judge-scripts

Commit `r161:d5a0f0a406b5` public

parent | child

Description:

clean up run command to check for running time / memory merge check.float

Commit status:

[Not Reviewed]

References:

merge algo

Diff options:

| | | |

Comments:

0 Commit comments 0 Inline Comments

Unresolved TODOs:

There are no unresolved TODOs

Add another comment

r161:d5a0f0a406b5 - Sat, 30 Aug 2014 11:21:22 - 4 files changed: 82 inserted, 9 deleted

templates/check.float created 755 ¶ +66

@@ -0,0 +1,66
	1	+ #!/usr/bin/env ruby
	2	+
	3	+ problem_home = ENV['PROBLEM_HOME']
	4	+ require "#{problem_home}/script/test_dsl.rb"
	5	+
	6	+ if ARGV.length < 2
	7	+ puts "Usage: check <language> <test-number> [<output-file>]"
	8	+ exit(0)
	9	+ end
	10	+
	11	+ language = ARGV[0]
	12	+ test_num = ARGV[1].to_i
	13	+ if ARGV.length >= 3
	14	+ output_file_name = ARGV[2]
	15	+ else
	16	+ output_file_name = "output.txt"
	17	+ end
	18	+
	19	+ load "#{problem_home}/test_cases/all_tests.cfg"
	20	+ problem = Problem.get_instance
	21	+
	22	+ output_file = File.new(output_file_name, "r")
	23	+ answer_file = File.new("#{problem_home}/test_cases/#{test_num}/answer-#{test_num}.txt")
	24	+ result_file = File.new("check_result", "w")
	25	+
	26	+ output_file_content = output_file.read
	27	+ answer_file_content = answer_file.read
	28	+
	29	+ report_correct = lambda {
	30	+ result_file.write "Correct\n"
	31	+ result_file.write problem.get_score(test_num)
	32	+ result_file.write "\n"
	33	+ result_file.close
	34	+ exit(0)
	35	+ }
	36	+
	37	+ report_wrong = lambda {
	38	+ result_file.write "Incorrect\n"
	39	+ result_file.write "0\n"
	40	+ result_file.close
	41	+ exit(0)
	42	+ }
	43	+
	44	+ ##################
	45	+ # Your code here #
	46	+ ##################
	47	+
	48	+ ########### THIS IS FOR CHECKING FLOAT with EPSILON error ##########
	49	+
	50	+ EPSILON = 0.000001
	51	+
	52	+ out_items = output_file_content.split
	53	+ ans_items = answer_file_content.split
	54	+
	55	+ if out_items.length != ans_items.length
	56	+ report_wrong.call
	57	+ else
	58	+ out_items.length.times do \|i\|
	59	+ out_value = out_items[i].to_f
	60	+ ans_value = ans_items[i].to_f
	61	+ if (out_value - ans_value).abs > EPSILON * [out_value.abs,ans_value.abs].max
	62	+ report_wrong.call
	63	+ end
	64	+ end
	65	+ report_correct.call
	66	+ end

std-script/compile ¶ +1

                 command = "#{PASCAL_COMPILER} #{params[:source_file]} -ooutpas #{PASCAL_OPTIONS} > #{params[:message_file]}"
                 system(command)
                 FileUtils.mv("output", params[:output_file])
               when "java"
                 #rename the file to the public class name
                 #get the class name
                 classname = 'DUMMY'
                 File.foreach(params[:source_file]) do |line|
                   md = /\s*public\s*class\s*(\w*)/.match(line)
                   classname=md[1] if md
                 end
                 system("cp #{params[:source_file]} #{classname}.java")
                 command = "#{JAVA_COMPILER} #{classname}.java 2> #{params[:message_file]}"
                 system(command)
                 if File.exists?(classname + ".class")
                   File.open(params[:output_file],"w") {|file| file.write("#{classname}")}
                 end
                 if classname == 'DUMMY'
                   File.open(params[:message_file],"w") {|file| file.write("Cannot find any public class in the source code\n")}
                 end
               when "ruby"
                 command = "#{RUBY_INTERPRETER} -c #{params[:source_file]} 2> #{params[:message_file]}"
                 if system(command)
                   File.open(params[:output_file],"w") do |out_file|
                     out_file.puts "#!#{RUBY_INTERPRETER}"
                     File.open(params[:source_file],"r").each do |line|
                       out_file.print line
                     end
                   end
                   File.chmod(0755, params[:output_file])
                 end
               when "python"
                 command = "#{PYTHON_CHECKER} #{params[:source_file]} > #{params[:message_file]}"
                 if system(command)
                   #compile to python bytecode
                   command = "#{PYTHON_INTERPRETER} -m py_compile #{params[:source_file]}"
                   puts "compile: #{command}"
                   system(command)
                   puts "pwd: " + Dir.pwd
                   Dir.new('.').each {|file| puts file}
                   File.open(params[:output_file],"w") do |out_file|
                     out_file.puts "#!#{PYTHON_INTERPRETER} #{params[:source_file]}c"
                   end
                   File.chmod(0755, params[:output_file])
+            +     FileUtils.cp("#{params[:source_file]}c",params[:output_file])
                 end
               else
                 talk("ERROR: Invalid language specified!")
                 open(params[:message_file],"w") do |f|
                   f.puts "ERROR: Invalid language specified!"
                 end
                 exit(127)
               end
               # Report success or failure.
               if FileTest.exists? params[:output_file]
                 talk "Compilation was successful!"
               else
                 talk "ERROR: Something was wrong during the compilation!"
               end

std-script/judge ¶ +2 -1

                 end
               end
               problem_home = ENV['PROBLEM_HOME']
               def execute(command, error_message="")
                 if not system(command)
                   msg = "ERROR: #{error_message}"
                   log msg
                   raise(msg)
                 end
               end
               def call_and_log(error_message)
                 begin
                   yield
                 rescue
                   msg = "ERROR: #{error_message}"
                   log msg
                   raise msg
                 end
               end
               def clear_and_create_empty_dir(dir)
                 FileUtils.rm_rf(dir, :secure => true)
                 call_and_log("Cannot make directory #{dir}.") { FileUtils.mkdir(dir) }
               end
               # ARGV[0] --- language
               # ARGV[1] --- program source file
               # ARGV[2] --- test result directory
               # ARGV[3] --- sandbox directory
               if ARGV.length < 2 || ARGV.length > 4
                 puts "Usage: judge <language> <program-source> [<test-result-directory>] [<sandbox-directory>]"
                 puts "  <sandbox-directory> is defaulted to ./sandbox"
                 puts "  <test-result-directory> is defaulted to ./test-result"
                 puts "WARNING: The judge script will forcefully create the (implicitly and explicitly) specified directories and remove anything inside it."
                 exit(127)
               end
               language = ARGV[0]
               if language != "c" && language != "c++" && language != "pas" && language != "java" && language != "ruby" && language != "python"
                 log "You specified a language that is not supported: #{language}."
                 exit(127)
               end
               source_file = ARGV[1]
+            + ENV['SOURCE_NAME'] = source_file
               if File.exist?(source_file) == false
                 log "The source file does not exist."
                 exit(127)
               end
               log "Making test result and sandbox directories..."
               current_dir = FileUtils.pwd
               current_dir.strip!
               if ARGV.length >= 3
                 test_result_dir = ARGV[2]
               else
                 test_result_dir = "#{current_dir}/test-result"
               end
               log "Test result directory: #{test_result_dir}"
               clear_and_create_empty_dir(test_result_dir)
               if ARGV.length >= 4
                 sandbox_dir = ARGV[3]
               else
                 sandbox_dir = "#{current_dir}/sandbox"
               end
               log "Sandbox directory: #{sandbox_dir}"
               clear_and_create_empty_dir(sandbox_dir)
               # Compile
               log
               log "Compiling..."
               call_and_log("Cannot copy the source file to #{sandbox_dir}") {
                 FileUtils.cp(source_file, sandbox_dir)
               }
               begin
                 Dir.chdir sandbox_dir
               rescue
                 log "ERROR: Cannot change directory to #{sandbox_dir}."
                 exit(127)
               end
               execute("#{problem_home}/script/compile #{language} #{source_file}", "Compilation error!")
               compile_message = open("compiler_message").read
               compile_message.strip!
               call_and_log("Cannot move the compiler message to #{test_result_dir}.") {
                 FileUtils.mv("compiler_message", test_result_dir)
               }
               if !FileTest.exist?("a.out")
                 log "Cannot compile the source code. See message in #{test_result_dir}/compile_message"
                 exit(127)
               else
                 call_and_log("Cannot move the compiled program to #{test_result_dir}") {
                   FileUtils.mv("a.out",test_result_dir)
                   if language == "java" then Dir["*.class"].each { |file| FileUtils.mv(file,test_result_dir)} end
                   if language == "python" then Dir["*.pyc"].each { |file| FileUtils.mv(file,test_result_dir)} end
                 }
                 FileUtils.rm_rf("#{sandbox_dir}/.")
               end
               require "#{problem_home}/script/test_dsl.rb"
               load "#{problem_home}/test_cases/all_tests.cfg"
               problem = Problem.get_instance
               if problem.well_formed? == false
                 log "The problem specification is not well formed."
                 exit(127)
               end
               # Doing the testing.
               (1..(problem.num_tests)).each do |test_num|
                 $stdout.print "[#{test_num}]"
                 $stdout.flush
                 log "Test number: #{test_num}"
                 call_and_log("Cannot copy the compiled program into #{sandbox_dir}") {
                   FileUtils.cp("#{test_result_dir}/a.out", sandbox_dir, :preserve => true)
                   if language == "java" then Dir["#{test_result_dir}/*.class"].each { |file| FileUtils.cp(file,sandbox_dir)} end
                   if language == "python" then Dir["#{test_result_dir}/*.pyc"].each { |file| FileUtils.cp(file,sandbox_dir)} end
                 }
                 begin
-            -     execute("#{problem_home}/script/run #{language} #{test_num}", "Error occured during execution of the run script")
+            +     execute("#{problem_home}/script/run #{language} #{test_num} ", "Error occured during execution of the run script")
                 rescue
                   # do nothing
                 end
                 call_and_log("Cannot create directory #{test_result_dir}/#{test_num}") {
                   FileUtils.mkdir "#{test_result_dir}/#{test_num}"
                 }
                 call_and_log("Cannot copy the result file into #{test_result_dir}/#{test_num}") {
                   FileUtils.mv "#{sandbox_dir}/result", "#{test_result_dir}/#{test_num}"
                 }
                 call_and_log("Cannot copy the comment file into #{test_result_dir}/#{test_num}") {
                   FileUtils.mv "#{sandbox_dir}/comment", "#{test_result_dir}/#{test_num}"
                 }
                 call_and_log("Cannot copy the output file into #{test_result_dir}/#{test_num}") {
                   FileUtils.mv "#{sandbox_dir}/output.txt", "#{test_result_dir}/#{test_num}"
                 }
                 call_and_log("Cannot clear #{sandbox_dir}") {
                   FileUtils.rm_rf(Dir.glob("#{sandbox_dir}/*"), :secure => true)
                 }
               end
               $stdout.print "[done]\n"
               # Grade
               log
               log "Grading..."
               begin
                 Dir.chdir test_result_dir
               rescue
                 log "ERROR: Cannot change directory to #{test_result_dir}."
                 exit(127)
               end
               execute("#{problem_home}/script/grade", "An error occured during grading!")
               log
               log "All done!"

std-script/run ¶ +13 -8

               #!/usr/bin/env ruby
               require 'fileutils'
               def log(str='')
                 if ENV['TALKATIVE']!=nil
                   puts str
                 end
                 if ENV['GRADER_LOGGING']!=nil
                   log_fname = ENV['GRADER_LOGGING']
                   fp = File.open(log_fname,"a")
                   fp.puts("run: #{Time.new.strftime("%H:%M")} #{str}")
                   fp.close
                 end
               end
               def extract_time(t)
                 #  puts "TIME: #{t}"
                 if (result=/^(.*)r(.*)u(.*)s/.match(t))
                   {:real => result[1], :user => result[2], :sys => result[3]}
                 else
                   #{:real => 0, :user => 0, :sys => 0}
                   #puts "ERROR READING RUNNING TIME: #{t}"
                   raise "Error reading running time: #{t}"
                 end
               end
               def compile_box(source,bin)
                 system("g++ #{source} -o #{bin}")
               end
               if ARGV.length < 2 || ARGV.length > 3
                 puts "Usage: run <language> <test-num> [<program-name>]"
                 exit(127)
               end
               language = ARGV[0]
               test_num = ARGV[1].to_i
               if ARGV.length > 2
                 program_name = ARGV[2]
               else
                 program_name = "a.out"
               end
               problem_home = ENV['PROBLEM_HOME']
+            + source_name = ENV['SOURCE_NAME']
               require "#{problem_home}/script/test_dsl.rb"
               load "#{problem_home}/test_cases/all_tests.cfg"
               problem = Problem.get_instance
+            + sandbox_dir = Dir.getwd
+            +
               if problem.well_formed? == false
                 log "The problem specification is not well formed."
                 exit(127)
               end
               # Check if the test number is okay.
               if test_num <= 0 || test_num > problem.num_tests
                 log "You have specified a wrong test number."
                 exit(127)
               end
               #####################################
               # Set the relavant file names here. #
               #####################################
               input_file_name = "#{problem_home}/test_cases/#{test_num}/input-#{test_num}.txt"
               #####################################
               time_limit = problem.get_time_limit test_num
               mem_limit = problem.get_mem_limit(test_num) * 1024
               # Copy the input file.
               #`cp #{problem_home}/test_cases/#{test_num}/#{input_file_name} .`
               # check if box is there, if not, compile it!
               if !File.exists?("#{problem_home}/script/box")
                 log "WARNING: Compiling box: to increase efficiency, it should be compile manually"
                 compile_box("#{problem_home}/script/box.cc",
                             "#{problem_home}/script/box")
               end
               # Hide PROBLEM_HOME
               ENV['PROBLEM_HOME'] = nil
+            + ENV['SOURCE_NAME'] = nil
               # Run the program.
               #run_command = "/usr/bin/time -f \"#{time_output_format}\" 2>run_result #{problem_home}/script/box_new -a 2 -f -t #{time_limit} -m #{mem_limit} -i #{input_file_name} -o output.txt #{program_name}"
               #
+            + JAVA_OPTION = "-s set_robust_list -s futex -s clone -s getppid -s clone -s wait4 -p /usr/bin/ -p ./"
+            + RUBY_OPTION = "-p /usr/lib64/ -p /lib64/ -p /dev/urandom -p #{sandbox_dir}/#{program_name} -s set_robust_list -s sched_getaffinity -s clock_gettime -s sigaltstack -s pipe2 -s clone -s futex -s openat"
+            + PYTHON_OPTION = "-p /usr/lib64/ -p /lib64/ -p /usr/bin/ -p /usr/local/lib64/ -p /usr/local/lib/ -p #{sandbox_dir}/#{program_name} -p ./#{program_name} -p #{sandbox_dir}/#{source_name} -s set_robust_list -s openat -s recvmsg -s connect -s socket -s sendto -E PYTHONNOUSERSITE=yes"
               case language
                 when "java"
                   # for java, extract the classname
                   # wne have to add additional systemcall and we don't check the mem limit (dunno how to fix...)
                   classname = 'DUMMY'
                   File.open(program_name,"r").each do |line|
                     classname = line
                   end
-            -     run_command = "#{problem_home}/script/box -T -t #{time_limit} -s getppid -s clone -s wait4 -p /usr/bin/ -p ./ -i #{input_file_name} -o output.txt /usr/bin/java #{classname} 2>run_result"
+            +     #for java, we cannot really check the memory limit...
+            +     run_command = "#{problem_home}/script/box -a 3 -f -T -t #{time_limit} #{JAVA_OPTION} -i #{input_file_name} -o output.txt /usr/bin/java #{classname} 2>run_result"
                 when "ruby"
-            -     run_command = "#{problem_home}/script/box -T -t #{time_limit} -s getppid -s wait4 -s clone -s set_robust_list -s futex -s sigaltstack -p /dev/urandom -p ./ -p /home/dae/.rvm/rubies/ruby-1.9.2-p320/ -p #{problem_home}/ -i #{input_file_name} -o output.txt #{program_name} 2>run_result"
+            +     run_command = "#{problem_home}/script/box -a 2 -f -T -t #{time_limit} -m #{mem_limit} #{RUBY_OPTION} -i #{input_file_name} -o output.txt /usr/bin/ruby #{program_name} 2>run_result"
                 when "python"
-            -     #this code just run without any checking
+            +     run_command = "#{problem_home}/script/box -a 2 -f -T -t #{time_limit} -m #{mem_limit} #{PYTHON_OPTION} -i #{input_file_name} -o output.txt /usr/bin/python #{program_name} 2>run_result"
-            -     run_command = "#{problem_home}/script/box -T -t #{time_limit} -p #{problem_home}/ -i #{input_file_name} -o output.txt #{program_name} 2>run_result"
                 else # for c++, pascal, we do the normal checking
                   run_command = "#{problem_home}/script/box -a 2 -f -T -t #{time_limit} -m #{mem_limit} -i #{input_file_name} -o output.txt #{program_name} 2>run_result"
               end
               log "Running test #{test_num}..."
               log run_command
-            - log
+            + log
               system(run_command)
               # Restore PROBLEM_HOME
               ENV['PROBLEM_HOME'] = problem_home
               # Create the result file.
               result_file = File.new("result", "w")
               comment_file = File.new("comment", "w")
               # Check if the program actually produced any output.
               run_result_file = File.new("run_result", "r")
               run_result = run_result_file.readlines
               run_result_file.close
               run_stat = run_result[run_result.length-1]
               running_time = extract_time(run_stat)
               report = lambda{ |status, points, comment|
                 result_file.write status.strip
                 result_file.write "\n"
                 result_file.write points.to_s.strip
                 result_file.write "\n"
                 result_file.write run_stat.strip
                 result_file.write "\n"
                 result_file.close
                 FileUtils.rm "run_result"
                 # `rm output.txt`   --- keep the output
                 comment_file.write comment
                 # added for debuggin --- jittat
                 comment_file.write "--run-result--\n"
                 run_result.each do |l|
                   comment_file.write l
                 end
                 comment_file.close
                 log "Done!"
                 exit(0)
               }
+            +
               if run_result[0][0,2] != "OK"
                 log "There was a runtime error."
                 report.call(run_result[0], 0, "No comment.\n")
               end
               if running_time[:user].to_f > time_limit
                 log "Time limit exceeded."
                 report.call("Time limit exceeded", 0, "No comment.\n")
               end
               # Run 'check' to evaluate the output.
               #puts "There was no runtime error. Proceed to checking the output."
               check_command = "#{problem_home}/script/check #{language} #{test_num}"
               log "Checking the output..."
               log check_command
               if not system(check_command)
                 log "Problem with check script"
                 report.call("Incorrect",0,"Check script error.\n")
                 exit(127)
               end
               check_file = File.new("check_result", "r")
               check_file_lines = check_file.readlines
               report.call(check_file_lines[0], check_file_lines[1], "No comment.\n")

Write
Preview

You need to be logged in to leave comments. Login now

	Site-wide shortcuts
/	Use quick search box
g h	Goto home page
g g	Goto my private gists page
g G	Goto my public gists page
n r	New repository page
n g	New gist page

	Repositories
g s	Goto summary page
g c	Goto changelog page
g f	Goto files page
g F	Goto files page with file search activated
g p	Goto pull requests page
g o	Goto repository settings
g O	Goto repository permissions settings

Sign in to your account

Author

r161:d5a0f0a406b5 - Sat, 30 Aug 2014 11:21:22 - 4 files changed: 82 inserted, 9 deleted

Sign in to your account

Commit r161:d5a0f0a406b5 public

Author

r161:d5a0f0a406b5 - Sat, 30 Aug 2014 11:21:22 - 4 files changed: 82 inserted, 9 deleted

Commit `r161:d5a0f0a406b5` public