cafe-grader-judge-scripts Commit

cafe-grader-judge-scripts

Commit `r189:45a8ea0792f3` public

parent | child

Description:

big merge with dae

Commit status:

[Not Reviewed]

References:

merge default

Diff options:

| | | |

Comments:

0 Commit comments 0 Inline Comments

Unresolved TODOs:

There are no unresolved TODOs

Add another comment

r189:45a8ea0792f3 - Thu, 05 Feb 2015 03:19:35 - 12 files changed: 267 inserted, 31 deleted

templates/check.float created 755 ¶ +66

@@ -0,0 +1,66
	1	+ #!/usr/bin/env ruby
	2	+
	3	+ problem_home = ENV['PROBLEM_HOME']
	4	+ require "#{problem_home}/script/test_dsl.rb"
	5	+
	6	+ if ARGV.length < 2
	7	+ puts "Usage: check <language> <test-number> [<output-file>]"
	8	+ exit(0)
	9	+ end
	10	+
	11	+ language = ARGV[0]
	12	+ test_num = ARGV[1].to_i
	13	+ if ARGV.length >= 3
	14	+ output_file_name = ARGV[2]
	15	+ else
	16	+ output_file_name = "output.txt"
	17	+ end
	18	+
	19	+ load "#{problem_home}/test_cases/all_tests.cfg"
	20	+ problem = Problem.get_instance
	21	+
	22	+ output_file = File.new(output_file_name, "r")
	23	+ answer_file = File.new("#{problem_home}/test_cases/#{test_num}/answer-#{test_num}.txt")
	24	+ result_file = File.new("check_result", "w")
	25	+
	26	+ output_file_content = output_file.read
	27	+ answer_file_content = answer_file.read
	28	+
	29	+ report_correct = lambda {
	30	+ result_file.write "Correct\n"
	31	+ result_file.write problem.get_score(test_num)
	32	+ result_file.write "\n"
	33	+ result_file.close
	34	+ exit(0)
	35	+ }
	36	+
	37	+ report_wrong = lambda {
	38	+ result_file.write "Incorrect\n"
	39	+ result_file.write "0\n"
	40	+ result_file.close
	41	+ exit(0)
	42	+ }
	43	+
	44	+ ##################
	45	+ # Your code here #
	46	+ ##################
	47	+
	48	+ ########### THIS IS FOR CHECKING FLOAT with EPSILON error ##########
	49	+
	50	+ EPSILON = 0.000001
	51	+
	52	+ out_items = output_file_content.split
	53	+ ans_items = answer_file_content.split
	54	+
	55	+ if out_items.length != ans_items.length
	56	+ report_wrong.call
	57	+ else
	58	+ out_items.length.times do \|i\|
	59	+ out_value = out_items[i].to_f
	60	+ ans_value = ans_items[i].to_f
	61	+ if (out_value - ans_value).abs > EPSILON * [out_value.abs,ans_value.abs].max
	62	+ report_wrong.call
	63	+ end
	64	+ end
	65	+ report_correct.call
	66	+ end

grader ¶ +13 -1

                        the problem name must be specified by the next argument.
                        additional options:
+            +             --all-sub    re-grade every submissions instead of just the latest submission of each user.
-            -             --all-sub    re-grade every submissions instead of just the latest submission of each user.
                 sub:   re-grader the specified submission.
                        The submission ID to be re-graded must be specified by the next argument.
+            +   options:
+            +     --err-log   log error to a file in the log dir
+            +
               (3) create stop-file to stop running grader in queue mode
               (4) You are here.
               USAGE
                 options[:all_sub] = (ARGV.delete('--all-sub') != nil)
+            +   options[:err_log] = (ARGV.delete('--err-log') != nil)
+            +
                 return options
               end
               #set loggin environment
               ENV['GRADER_LOGGING'] = log_file_name
+            + if options[:err_log]
+            +   err_file_name = log_file_name + '.err'
+            +   $stderr.reopen(err_file_name,"a")
+            +   log "STDERR log to file [#{err_file_name}]"
+            +   warn "start logging for grader PID #{Process.pid} on #{Time.now.in_time_zone}"
+            + end
+            +
               # register exit handler to report inactive, and terminated
               at_exit do

lib/engine.rb ¶ +3 -1

                     talk grading_dir
                     Dir.chdir grading_dir
-            -       cmd = "#{problem_home}/script/judge #{language} #{fname}"
+            +       script_name = "#{problem_home}/script/judge"
+            +       cmd = "#{script_name} #{language} #{fname}"
                     talk "CMD: #{cmd}"
+            +       warn "ERROR: file does not exists #{script_name}" unless File.exists? script_name
                     system(cmd)
                   end

lib/runner.rb ¶ +1 -1

                   end
                   def grade_submission(submission)
-            -       puts "Submission: #{submission.id} by #{submission.user.full_name}"
+            +       puts "Submission: #{submission.id} by #{submission.try(:user).try(:full_name)}"
                     @engine.grade(submission)
                   end

lib/submission_helper.rb ¶ +22 -4

                     end
                     result_fname = "#{test_result_dir}/result"
-            -       comment_fname = "#{test_result_dir}/comment"
+            +       comment_fname = "#{test_result_dir}/comment"
+            +       runstat_fname = "#{test_result_dir}/run_stat"
                     if FileTest.exist?(result_fname)
                       comment = ""
                       begin
                         comment += ""
                       end
-            -         return {:points => result,
+            +         begin
-            -           :comment => comment,
+            +           runstat_file = File.open(runstat_fname)
-            -           :cmp_msg => cmp_msg}
+            +           max_runtime = runstat_file.readline.to_f
+            +           peak_memory = runstat_file.readline.to_i
+            +         rescue
+            +           max_runtime = -1
+            +           peak_memory = -1
+            +         end
+            +
+            +
+            +         return {points: result,
+            +           comment: comment,
+            +           cmp_msg: cmp_msg,
+            +           max_runtime: max_runtime,
+            +           peak_memory: peak_memory
+            +         }
                     else
                       if FileTest.exist?("#{test_result_dir}/a.out")
                         return {:points => 0,
                     submission.points = points
                     comment = @config.report_comment(result[:comment])
+            +       submission.peak_memory = result[:peak_memory]
+            +       submission.max_runtime =  result[:max_runtime]
+            +       submission.effective_code_length =submission.source.length
+            +
                     #
                     # TODO: FIX THIS MESSAGE
                     #

lib/test_request_helper.rb ¶ +1 -1

                     end
                     # extract memory usage
-            -       if res = /s(.*)m/.match(running_stat_line)
+            +       if res = /s(.*)kbytes/.match(running_stat_line)
                       memory_used = res[1].to_i
                     else
                       memory_used = -1

std-script/compile ¶ +89 -14

@@ -26,10 +26,18
26	C_COMPILER = "/usr/bin/gcc"	26	C_COMPILER = "/usr/bin/gcc"
27	CPLUSPLUS_COMPILER = "/usr/bin/g++"	27	CPLUSPLUS_COMPILER = "/usr/bin/g++"
28	PASCAL_COMPILER = "/usr/bin/fpc"	28	PASCAL_COMPILER = "/usr/bin/fpc"
		29	+ JAVA_COMPILER = "/usr/bin/javac"
		30	+ RUBY_INTERPRETER = "/usr/bin/ruby"
		31	+ PYTHON_INTERPRETER = "/usr/bin/python"
		32	+ PYTHON_CHECKER = "/usr/bin/pyflakes"
		33	+ PHP_INTERPRETER = "/usr/bin/php"
29		34
30	C_OPTIONS = "-O2 -s -static -std=c99 -DCONTEST -lm -Wall"	35	C_OPTIONS = "-O2 -s -static -std=c99 -DCONTEST -lm -Wall"
31	- CPLUSPLUS_OPTIONS = "-O2 -s -static -DCONTEST -lm -Wall"	36	+ CPLUSPLUS_OPTIONS = "-O2 -s -std=c++11 -static -DCONTEST -lm -Wall"
32	PASCAL_OPTIONS = "-O1 -XS -dCONTEST"	37	PASCAL_OPTIONS = "-O1 -XS -dCONTEST"
		38	+ JAVA_OPTIONS = ""
		39	+ PYTHON_OPTIONS = ""
		40	+ PHP_OPTIONS = "-l"
33		41
34	# Check for the correct number of arguments. Otherwise, print usage.	42	# Check for the correct number of arguments. Otherwise, print usage.
35	if ARGV.length == 0 or ARGV.length > 4	43	if ARGV.length == 0 or ARGV.length > 4
@@ -84,20 +92,87
84	# Compile.	92	# Compile.
85	case params[:prog_lang]	93	case params[:prog_lang]
86		94
87	- when "c"	95	+ when "c"
88	- command = "#{C_COMPILER} #{params[:source_file]} -o #{params[:output_file]} #{C_OPTIONS} ~~2> #{params[:message_file]}~~"	96	+ command = "#{C_COMPILER} #{params[:source_file]} -o #{params[:output_file]} #{C_OPTIONS}"
89	- system(command)	97	+ system(command, err: params[:message_file])
		98	+
		99	+ when "c++"
		100	+ command = "#{CPLUSPLUS_COMPILER} #{params[:source_file]} -o #{params[:output_file]} #{CPLUSPLUS_OPTIONS}"
		101	+ system(command, err: params[:message_file])
		102	+
		103	+ when "pas"
		104	+ command = "#{PASCAL_COMPILER} #{params[:source_file]} -ooutpas #{PASCAL_OPTIONS}"
		105	+ system(command,out: params[:message_file])
		106	+ FileUtils.mv("output", params[:output_file])
		107	+
		108	+ when "java"
		109	+ #rename the file to the public class name
		110	+
		111	+ #get the class name
		112	+ classname = 'DUMMY'
		113	+ source = Array.new
		114	+ File.foreach(params[:source_file],'r:UTF-8') do \|line\|
		115	+ line.encode!('UTF-8','UTF-8',invalid: :replace, replace: '')
		116	+ md = /\spublic\sclass\s(\w)/.match(line)
		117	+ classname=md[1] if md
		118	+ source << line unless line =~ /\spackage\s\w+\s*\;/
		119	+ end
		120	+ File.open("#{classname}.java","w") do \|file\|
		121	+ source.each do \|s\|
		122	+ file.puts s
		123	+ end
		124	+ end
		125	+ #system("cp #{params[:source_file]} #{classname}.java")
		126	+ command = "#{JAVA_COMPILER} -encoding utf8 #{classname}.java"
		127	+ system(command, err: params[:message_file])
		128	+ if File.exists?(classname + ".class")
		129	+ File.open(params[:output_file],"w") {\|file\| file.write("#{classname}")}
		130	+ end
		131	+ if classname == 'DUMMY'
		132	+ File.open(params[:message_file],"w") {\|file\| file.write("Cannot find any public class in the source code\n")}
		133	+ end
90		134
91	- when "~~c++~~"	135	+ when "ruby"
92	- command = "#{CPLUSPLUS_COMPILER} #{params[:source_file]} -o #{params[:output_file]} #{CPLUSPLUS_OPTIONS} 2> #{params[:message_file]}"	136	+ command = "#{RUBY_INTERPRETER} -c #{params[:source_file]}"
93	- system(command)	137	+ if system(command, err: params[:message_file])
94	-	138	+ File.open(params[:output_file],"w") do \|out_file\|
95	- when "pas"	139	+ out_file.puts "#!#{RUBY_INTERPRETER}"
96	- command = "#{PASCAL_COMPILER} #{params[:source_file]} -ooutpas #{PASCAL_OPTIONS} > #{params[:message_file]}"	140	+ File.open(params[:source_file],"r").each do \|line\|
97	- system(command)	141	+ out_file.print line
98	- FileUtils.mv("output", params[:output_file])	142	+ end
99	-	143	+ end
100	- else	144	+ File.chmod(0755, params[:output_file])
		145	+ end
		146	+
		147	+ when "python"
		148	+ command = "#{PYTHON_CHECKER} #{params[:source_file]}"
		149	+ if system(command, out: params[:message_file])
		150	+ #compile to python bytecode
		151	+ command = "#{PYTHON_INTERPRETER} -m py_compile #{params[:source_file]}"
		152	+ puts "compile: #{command}"
		153	+ system(command)
		154	+ puts "pwd: " + Dir.pwd
		155	+ Dir.new('.').each {\|file\| puts file}
		156	+ File.open(params[:output_file],"w") do \|out_file\|
		157	+ out_file.puts "#!#{PYTHON_INTERPRETER} #{params[:source_file]}c"
		158	+ end
		159	+ File.chmod(0755, params[:output_file])
		160	+ FileUtils.cp("#{params[:source_file]}c",params[:output_file])
		161	+ end
		162	+
		163	+ when "php"
		164	+ command = "#{PHP_INTERPRETER} #{PHP_OPTIONS} #{params[:source_file]}"
		165	+ if system(command, err: params[:message_file])
		166	+ File.open(params[:output_file],"w") do \|out_file\|
		167	+ out_file.puts "#!#{PHP_INTERPRETER}"
		168	+ File.open(params[:source_file],"r").each do \|line\|
		169	+ out_file.print line
		170	+ end
		171	+ end
		172	+ File.chmod(0755, params[:output_file])
		173	+ end
		174	+
		175	+ else
101	talk("ERROR: Invalid language specified!")	176	talk("ERROR: Invalid language specified!")
102	open(params[:message_file],"w") do \|f\|	177	open(params[:message_file],"w") do \|f\|
103	f.puts "ERROR: Invalid language specified!"	178	f.puts "ERROR: Invalid language specified!"

std-script/grade ¶ +29 -1

                 end
               end
+            + def extract_time(t)
+            +   #puts "TIME: #{t}"
+            +   if (result=/^(.*)r(.*)u(.*)s(.*)kbytes/.match(t))
+            +     {:real => result[1], :user => result[2], :sys => result[3], :mem => result[4]}
+            +   else
+            +     #{:real => 0, :user => 0, :sys => 0}
+            +     #puts "ERROR READING RUNNING TIME: #{t}"
+            +     raise "Error reading running time: #{t}"
+            +   end
+            + end
+            +
               problem_home = ENV['PROBLEM_HOME']
               require "#{problem_home}/script/test_dsl.rb"
               load "#{problem_home}/test_cases/all_tests.cfg"
               all_score = 0
               all_comment = ''
+            + peak_memory = -1
+            + max_runtime = -1
               (1..(problem.runs.length-1)).each do |k|
                 log "grade run #{k}"
                 run = problem.runs[k]
                   else
                     result_file = File.new(result_file_name, "r")
                     result_file_lines = result_file.readlines
-            -       if result_file_lines.length>=2
+            +       if result_file_lines.length>=3
                       current_run_score = result_file_lines[1].to_i
                       run_comment += result_file_lines[0]
                       run_comment_short += char_comment(result_file_lines[0].chomp)
+            +
+            +         #update max runtime & memory
+            +         run_stat = extract_time result_file_lines[2]
+            +         peak_memory = [peak_memory,run_stat[:mem].to_i].max
+            +         max_runtime = [max_runtime,run_stat[:user].to_f + run_stat[:sys].to_f].max
                     else
                       current_run_score = 0
                       run_comment += "result file for test #{test_num} error\n"
               comment_file = File.new("comment", "w")
               comment_file.write "#{all_comment}\n"
               comment_file.close
+            +
+            +
+            + File.open("run_stat","w") do |file|
+            +   file.puts max_runtime
+            +   file.puts peak_memory
+            + end
+            +
+            + puts "#{all_score} #{all_comment}"
+            + log "score = #{all_score}\ncomment = #{all_comment}"
+            + log "max_runtime = #{max_runtime}\npeak_memory = #{peak_memory}"

std-script/judge ¶ +7 -2

               end
               language = ARGV[0]
-            - if language != "c" && language != "c++" && language != "pas"
+            + if language != "c" && language != "c++" && language != "pas" && language != "java" && language != "ruby" && language != "python" && language != "php"
                 log "You specified a language that is not supported: #{language}."
                 exit(127)
               end
               source_file = ARGV[1]
+            + ENV['SOURCE_NAME'] = source_file
               if File.exist?(source_file) == false
                 log "The source file does not exist."
                 exit(127)
               else
                 call_and_log("Cannot move the compiled program to #{test_result_dir}") {
                   FileUtils.mv("a.out",test_result_dir)
+            +     if language == "java" then Dir["*.class"].each { |file| FileUtils.mv(file,test_result_dir)} end
+            +     if language == "python" then Dir["*.pyc"].each { |file| FileUtils.mv(file,test_result_dir)} end
                 }
                 FileUtils.rm_rf("#{sandbox_dir}/.")
               end
                 call_and_log("Cannot copy the compiled program into #{sandbox_dir}") {
                   FileUtils.cp("#{test_result_dir}/a.out", sandbox_dir, :preserve => true)
+            +     if language == "java" then Dir["#{test_result_dir}/*.class"].each { |file| FileUtils.cp(file,sandbox_dir)} end
+            +     if language == "python" then Dir["#{test_result_dir}/*.pyc"].each { |file| FileUtils.cp(file,sandbox_dir)} end
                 }
                 begin
-            -     execute("#{problem_home}/script/run #{language} #{test_num}", "Error occured during execution of the run script")
+            +     execute("#{problem_home}/script/run #{language} #{test_num} ", "Error occured during execution of the run script")
                 rescue
                   # do nothing
                 end

std-script/run ¶ +34 -4

               end
               problem_home = ENV['PROBLEM_HOME']
+            + source_name = ENV['SOURCE_NAME']
               require "#{problem_home}/script/test_dsl.rb"
               load "#{problem_home}/test_cases/all_tests.cfg"
               problem = Problem.get_instance
+            + sandbox_dir = Dir.getwd
+            +
               if problem.well_formed? == false
                 log "The problem specification is not well formed."
                 exit(127)
               # Hide PROBLEM_HOME
               ENV['PROBLEM_HOME'] = nil
+            + ENV['SOURCE_NAME'] = nil
               # Run the program.
               #run_command = "/usr/bin/time -f \"#{time_output_format}\" 2>run_result #{problem_home}/script/box_new -a 2 -f -t #{time_limit} -m #{mem_limit} -i #{input_file_name} -o output.txt #{program_name}"
+            + #
-            - run_command = "#{problem_home}/script/box -a 2 -f -T -t #{time_limit} -m #{mem_limit} -i #{input_file_name} -o output.txt #{program_name} 2>run_result"
+            + JAVA_OPTION = "-s set_robust_list -s futex -s clone -s getppid -s clone -s wait4 -p /usr/bin/ -p ./"
+            + RUBY_OPTION   = "-p /usr/lib64/ -p /usr/local/lib64/ -p /usr/local/lib/ -p /lib64/ -p /dev/urandom -p #{sandbox_dir}/#{program_name} -p #{sandbox_dir}/ -s set_robust_list -s sched_getaffinity -s clock_gettime -s sigaltstack -s pipe2 -s clone -s futex -s openat -s pipe"
+            + PYTHON_OPTION = "-p /usr/lib64/ -p /usr/local/lib64/ -p /usr/local/lib/ -p /usr/bin/ -p /lib64/ -p #{sandbox_dir}/#{program_name} -p ./#{program_name} -p #{sandbox_dir}/#{source_name} -s set_robust_list -s openat -s recvmsg -s connect -s socket -s sendto -s futex -E PYTHONNOUSERSITE=yes"
+            + PHP_OPTION = "-p /usr/lib64/ -p/lib64/ -p /usr/bin/ -p #{sandbox_dir}/#{program_name} -p ./#{program_name} -p /usr/share/ -s setfsuid -s setfsgid -s openat -s set_robust_list -s futex -s clone -s socket -s connect"
+            +
+            + case language
+            +   when "java"
+            +     # for java, extract the classname
+            +     # wne have to add additional systemcall and we don't check the mem limit (dunno how to fix...)
+            +     classname = 'DUMMY'
+            +     File.open(program_name,"r").each do |line|
+            +       classname = line
+            +     end
+            +     #for java, we cannot really check the memory limit...
+            +     run_command = "#{problem_home}/script/box -a 3 -f -T -t #{time_limit} #{JAVA_OPTION} -i #{input_file_name} -o output.txt /usr/bin/java -A -Xmx#{mem_limit}k -A #{classname} "
+            +   when "ruby"
+            +     run_command = "#{problem_home}/script/box -a 2 -f -T -t #{time_limit*=2} -m #{mem_limit} #{RUBY_OPTION} -i #{input_file_name} -o output.txt /usr/bin/ruby #{program_name} "
+            +   when "python"
+            +     run_command = "#{problem_home}/script/box -a 2 -f -T -t #{time_limit*=2} -m #{mem_limit} #{PYTHON_OPTION} -i #{input_file_name} -o output.txt /usr/bin/python #{program_name} "
+            +   when "php"
+            +     run_command = "#{problem_home}/script/box -a 2 -f -T -t #{time_limit*=2} -m #{[128*1024,mem_limit].max} #{PHP_OPTION} -i #{input_file_name} -o output.txt /usr/bin/php -A -d -A memory_limit=#{mem_limit}k -A #{program_name} "
+            +   else # for c++, pascal, we do the normal checking
+            +     run_command = "#{problem_home}/script/box -a 2 -f -T -t #{time_limit} -m #{mem_limit} -i #{input_file_name} -o output.txt #{program_name} "
+            + end
+            +
               log "Running test #{test_num}..."
               log run_command
-            - log
+            + log
-            - system(run_command)
+            + system(run_command,err: 'run_result')
               # Restore PROBLEM_HOME
               ENV['PROBLEM_HOME'] = problem_home
                 exit(0)
               }
+            +
               if run_result[0][0,2] != "OK"
                 log "There was a runtime error."
                 report.call(run_result[0], 0, "No comment.\n")
               end
-            - if running_time[:user].to_f + running_time[:sys].to_f > time_limit
+            + if running_time[:user].to_f > time_limit
                 log "Time limit exceeded."
                 report.call("Time limit exceeded", 0, "No comment.\n")
               end

test/data/ev/test_yesno/script/check ¶ +1 -1

-            - #!/usr/bin/ruby
+            + #!/usr/bin/env ruby
               problem_home = ENV['PROBLEM_HOME']
               require "#{problem_home}/script/test_dsl.rb"

test/data/test_request/problems/test_normal/script/check ¶ +1 -1

-            - #!/usr/bin/ruby
+            + #!/usr/bin/env ruby
               problem_home = ENV['PROBLEM_HOME']
               require "#{problem_home}/script/test_dsl.rb"

Write
Preview

You need to be logged in to leave comments. Login now

	Site-wide shortcuts
/	Use quick search box
g h	Goto home page
g g	Goto my private gists page
g G	Goto my public gists page
n r	New repository page
n g	New gist page

	Repositories
g s	Goto summary page
g c	Goto changelog page
g f	Goto files page
g F	Goto files page with file search activated
g p	Goto pull requests page
g o	Goto repository settings
g O	Goto repository permissions settings

Sign in to your account

Author

r189:45a8ea0792f3 - Thu, 05 Feb 2015 03:19:35 - 12 files changed: 267 inserted, 31 deleted

Sign in to your account

Commit r189:45a8ea0792f3 public

Author

r189:45a8ea0792f3 - Thu, 05 Feb 2015 03:19:35 - 12 files changed: 267 inserted, 31 deleted

Commit `r189:45a8ea0792f3` public