cafe-grader-judge-scripts Commit

cafe-grader-judge-scripts

Commit `r241:19abb7c667e3` public

parent | child

Description:

merge

Commit status:

[Not Reviewed]

References:

merge default

Diff options:

| | | |

Comments:

0 Commit comments 0 Inline Comments

Unresolved TODOs:

There are no unresolved TODOs

Add another comment

r241:19abb7c667e3 - Sat, 17 Jun 2017 06:19:08 - 7 files changed: 124 inserted, 4 deleted

check_similar created 755 ¶ +93

@@ -0,0 +1,93
	1	+ #!/usr/bin/env ruby
	2	+
	3	+ def config
	4	+ Grader::Configuration.get_instance
	5	+ end
	6	+
	7	+ def display_manual
	8	+ puts <<USAGE
	9	+ Check similarity between submission
	10	+ using: check_similar sub1 sub2
	11	+ -- or --
	12	+ check_similar problem_name
	13	+ sub1 and sub2 are submission IDs to be checked
	14	+ problem_name will check all submissions of the problem wit problem short name is 'problem_name'
	15	+
	16	+ The output are given as
	17	+ sub1.login, sub1.id, sub1.point, sub2.login, sub2.id, sub2.point, similarity
	18	+
	19	+ USAGE
	20	+ end
	21	+
	22	+ def process_options_and_stop_file
	23	+
	24	+ # Process 'help' option
	25	+ if (ARGV.length == 0) or ((ARGV.length==1) and (/help/.match(ARGV[0])))
	26	+ display_manual
	27	+ exit(0)
	28	+ end
	29	+
	30	+ #default options
	31	+ options = {
	32	+ :dry_run => false,
	33	+ }
	34	+
	35	+
	36	+ if ARGV.length == 2
	37	+ options[:sub1] = ARGV[0].to_i
	38	+ options[:sub2] = ARGV[1].to_i
	39	+ elsif ARGV.length == 1
	40	+ options[:problem] = ARGV[0]
	41	+ end
	42	+
	43	+
	44	+ return options
	45	+ end
	46	+
	47	+ def compare(sub1,sub2,full = sub1.problem.full_score)
	48	+ dis = @jarow.getDistance(sub1.source, sub2.source)
	49	+ puts [sub1.user.login,"##{sub1.id}",(sub1.points * 100.0 / full).to_i,
	50	+ sub2.user.login,"##{sub2.id}",(sub2.points * 100.0 / full).to_i,
	51	+ "#{dis * 100}%"].join(',')
	52	+ end
	53	+
	54	+ #########################################
	55	+ # main program
	56	+ #########################################
	57	+
	58	+ options = process_options_and_stop_file
	59	+
	60	+ # load grader environment
	61	+ GRADER_ENV = 'grading'
	62	+ require File.join(File.dirname(__FILE__),'config/environment')
	63	+
	64	+ # boot rails, to be able to use the active record
	65	+ RAILS_ENV = config.rails_env
	66	+ require RAILS_ROOT + '/config/environment'
	67	+
	68	+ # load comparator
	69	+ require 'fuzzystringmatch'
	70	+ @jarow = FuzzyStringMatch::JaroWinkler.create( :native )
	71	+
	72	+ if options[:problem]
	73	+ p = Problem.where(name: options[:problem]).first
	74	+ unless p
	75	+ puts "cannot find problem #{options[:problem]}"
	76	+ exit(0)
	77	+ end
	78	+ subs = Submission.where(problem: p)
	79	+ full_score = p.full_score.to_i
	80	+ subs.each.with_index do \|s1,i\|
	81	+ puts "processing #{i+1} out of #{subs.length}"
	82	+ subs.each do \|s2\|
	83	+ if s1.user != s2.user
	84	+ compare(s1,s2,full_score)
	85	+ end
	86	+ end
	87	+ end
	88	+ else
	89	+ sub1 = Submission.find(options[:sub1])
	90	+ sub2 = Submission.find(options[:sub2])
	91	+ compare(sub1,sub2)
	92	+ end
	93	+

grader-process-check created 755 ¶ +6

@@ -0,0 +1,6
	1	+ #!/bin/bash
	2	+ count=`ps aux \| grep "cafe_grader" \| grep "grader grading queue" \| wc -l`
	3	+ if [ $count -lt 1 ]; then
	4	+ cd /home/dae/cafe_grader/judge
	5	+ /home/dae/.rvm/wrappers/ruby-2.3.0/ruby /home/dae/cafe_grader/judge/scripts/grader grading queue > /home/dae/grading.log &
	6	+ fi

import_problem ¶ +6

               end
               def print_usage
                 puts "using: import_problem_new name dir check [options]
               where: name = problem_name (put '-' (dash) to use dir name)
                      dir = importing testcase directory
                      check = check script, which can be
                              'integer', 'text' (for standard script),
                              path_to_your_script, or
                              'wrapper:(path_to_your_wrapped_script)'
               options: -t time-limit (in seconds)
                        -m memory-limit (in megabytes)
               The script looks at test data files in the dir of the forms: *.in and
               *.sol and import them to the evaluation dir for their environment,
               based on their prefixes.
               Currently supporting environments are:"
                 JUDGE_ENVIRONMENTS.each do |env|
                   prefix = ENV_INFO[env][:raw_prefix]
                   prefix = 'no prefix' if prefix==''
                   puts "  * #{env}"
                   puts "      import to: #{ENV_INFO[env][:ev_dir]}"
                   puts "      prefix with: #{prefix}  (e.g., #{ENV_INFO[env][:raw_prefix]}1.in, #{ENV_INFO[env][:raw_prefix]}5a.sol)"
                 end
                 puts"
               For each environment, the script
                 * creates a directory for a problem in ev dir of that environment,
                 * copies testdata in the old format and create standard testcase config file
                 * copies a check script for grading
                 * creates a test_request template in the ev dir + '/test_request'
               For wrapped checked script see comment in templates/check_wrapper for
                 information."
               end
               def count_testruns(testcase_dir, raw_prefix)
                 n = 0
                 begin
                   # check for test case n+1
                   if ((Dir["#{testcase_dir}/#{raw_prefix}#{n+1}.in"].length==0) and
                       (Dir["#{testcase_dir}/#{raw_prefix}#{n+1}[a-z].in"].length==0))
                     return n
                   end
                   n += 1
                 end while true
               end
               def create_dir_if_not_exists(dir)
                 if ! FileTest.exists? dir
                   FileUtils.mkdir(dir)
                 end
               end
               def import_problem(ev_dir, problem, testcase_dir, num_testruns, raw_prefix, check_script, options)
                 testrun_info = build_testrun_info_from_dir(num_testruns, testcase_dir, raw_prefix)
                 if !(FileTest.exists? ev_dir)
                   puts "Testdata dir (#{ev_dir}) not found."
                   return
                 end
                 problem_dir = "#{ev_dir}/#{problem}"
                 # start working
                 puts "creating directories"
                 create_dir_if_not_exists("#{problem_dir}")
                 create_dir_if_not_exists("#{problem_dir}/script")
                 create_dir_if_not_exists("#{problem_dir}/test_cases")
                 puts "copying testcases"
                 tr_num = 0
                 num_testcases = 0
                 testrun_info.each do |testrun|
                   tr_num += 1
                   puts "testrun: #{tr_num}"
                   testrun.each do |testcase_info|
                     testcase_num, testcase_fname = testcase_info
                     puts "copy #{testcase_fname} to #{testcase_num}"
                     create_dir_if_not_exists("#{problem_dir}/test_cases/#{testcase_num}")
                     copy_testcase("#{testcase_dir}",testcase_fname,"#{problem_dir}/test_cases/#{testcase_num}",testcase_num)
                     num_testcases += 1
                   end
                 end
+            +   #also include any .txt files
+            +   Dir.glob("#{testcase_dir}/*.txt") do |file|
+            +     puts "copy data file #{file}"
+            +     FileUtils.cp(file,"#{problem_dir}")
+            +   end
+            +
                 # generating all_tests.cfg
                 puts "generating testcase config file"
                 template = File.open(SCRIPT_DIR + "/templates/all_tests.cfg.erb").read
                 all_test_cfg = ERB.new(template)
                 cfg_file = File.open("#{problem_dir}/test_cases/all_tests.cfg","w")
                 cfg_file.puts all_test_cfg.result binding
                 cfg_file.close
                 # copy check script
                 if res = /^wrapper:(.*)$/.match(check_script)
                   # wrapper script
                   check_script_fname = res[1]
                   script_name = File.basename(check_script_fname)
                   check_wrapper_template = File.open(SCRIPT_DIR + "/templates/check_wrapper").read
                   check_wrapper = ERB.new(check_wrapper_template)
                   check_file = File.open("#{problem_dir}/script/check","w")
                   check_file.puts check_wrapper.result binding
                   check_file.close
                   File.chmod(0755,"#{problem_dir}/script/check")
                   FileUtils.cp("#{check_script_fname}", "#{problem_dir}/script/#{script_name}")
                 else
                   if File.exists?(SCRIPT_DIR + "/templates/check.#{check_script}")
                     check_script_fname = SCRIPT_DIR + "/templates/check.#{check_script}"
                   else
                     check_script_fname = check_script
                   end
                   FileUtils.cp("#{check_script_fname}", "#{problem_dir}/script/check", :preserve => true)
                 end
                 # generating test_request directory
                 puts "generating test_request template"
                 FileUtils.mkdir_p("#{ev_dir}/test_request/#{problem}/script")
                 FileUtils.mkdir_p("#{ev_dir}/test_request/#{problem}/test_cases/1")
                 template = File.open(SCRIPT_DIR + "/templates/test_request_all_tests.cfg.erb").read
                 test_request_all_test_cfg = ERB.new(template)
                 cfg_file = File.open("#{ev_dir}/test_request/#{problem}/test_cases/all_tests.cfg","w")
                 cfg_file.puts test_request_all_test_cfg.result
                 cfg_file.close
                 FileUtils.cp("#{SCRIPT_DIR}/templates/check_empty",
                              "#{ev_dir}/test_request/#{problem}/script/check")
                 FileUtils.cp("#{SCRIPT_DIR}/templates/answer-1.txt",
                              "#{ev_dir}/test_request/#{problem}/test_cases/1")
                 puts "done"
               end
               SCRIPT_DIR = File.dirname(__FILE__)
               # print usage
               if (ARGV.length < 3) or (ARGV[2][0,1]=="-")
                 print_usage
                 exit(127)
               end
               # processing arguments
               problem = ARGV[0]
               testcase_dir = ARGV[1]
               problem = File.basename(testcase_dir) if problem=="-"
               check_script = ARGV[2]
               options = {:time_limit => 1,  :mem_limit => 16}
               process_options(options)
               JUDGE_ENVIRONMENTS.each do |env|
                 ev_dir = ENV_INFO[env][:ev_dir]
                 raw_prefix = ENV_INFO[env][:raw_prefix]
                 num_testruns = count_testruns(testcase_dir,raw_prefix)
                 puts ""
                 puts "*** Environment: #{env} (#{num_testruns} test runs) ***"
                 puts ""
                 import_problem(ev_dir,
                                problem,
                                testcase_dir,
                                num_testruns,
                                raw_prefix,
                                check_script,
                                options)
               end

load_testcase ¶ +2 -2

               #!/usr/bin/env ruby
               def config
                 Grader::Configuration.get_instance
               end
               def display_manual
                 puts <<USAGE
               load_testcases
               using: load_testcases [problem_name ...]
                 problem_name    are list of "short name" of the problems
                 options:
                   --dry-run     do nothing, just simulate the run
                   --all         import all problem. This might take several minutes
               USAGE
               end
               def process_options_and_stop_file
                 # Process 'help' option
-            -   if (ARGV.length==1) and (/help/.match(ARGV[0]))
+            +   if (ARGV.length == 0) or ((ARGV.length==1) and (/help/.match(ARGV[0])))
                   display_manual
                   exit(0)
                 end
                 #default options
                 options = {
                   :dry_run => false,
                 }
                 options[:dry_run] = (ARGV.delete('--dry') != nil)
                 options[:all] = (ARGV.delete('--all') != nil)
                 return options
               end
               def process_problem(prob,dry_run = false)
                 prob.testcases.destroy_all
                 testcases_root = File.expand_path(GRADER_ROOT+"/../ev/#{prob.name}/test_cases/")
                 num = 1
                 puts "Processing problem #{prob.name}"
                 loop do
                   file_root = testcases_root + "/#{num}/"
                   puts "  checking file #{file_root}"
                   break unless File.exists? file_root
                   input = File.read(file_root + "/input-#{num}.txt")
                   answer = File.read(file_root + "/answer-#{num}.txt")
                   puts "  got test case ##{num} of size #{input.size} and #{answer.size}"
                   #THIS IS JUST A  PLACE HOLDER
                   group = num #this is wrong!!! fix it!!
                   score = 10
                   #BEWARE
                   prob.testcases.create(input: input,sol: answer, num: num, score:score,group: group) unless dry_run
                   num += 1
                 end
               end
               #########################################
               # main program
               #########################################
               options = process_options_and_stop_file
               # load grader environment
               GRADER_ENV = 'grading'
               require File.join(File.dirname(__FILE__),'config/environment')
               # boot rails, to be able to use the active record
               RAILS_ENV = config.rails_env
               require RAILS_ROOT + '/config/environment'
               if options[:all]
-            -   Problem.all.each { |prob| process_problem(prob,options[:all]) }
+            +   Problem.all.each { |prob| process_problem(prob,options[:dry_run]) }
               else
                 ARGV.each do |name|
                   prob = Problem.find_by(name: name)
                   process_problem(prob,options[:dry_run]) if prob
                   puts "Cannot find the problem #{name}" unless prob
                 end
               end

rename_problem ¶ +2 -2

               #!/usr/bin/env ruby
               ENVIRONMENT_DIRS = ['ev', 'ev-exam']
               def config
                 Grader::Configuration.get_instance
               end
               def rename_problem(old_problem_name, new_problem_name)
-            -   if valid_problem_name(new_problem_name)
+            +   unless valid_problem_name(new_problem_name)
                   puts "Bad new problem name: #{new_problem_name}"
                   return
                 end
                 problem = Problem.find_by_name(old_problem_name)
                 if problem==nil
                   puts "Problem #{old_problem_name} does not exist."
                   return
                 end
                 puts "Problem: #{old_problem_name} -> #{new_problem_name}"
                 ENVIRONMENT_DIRS.each do |dir|
                   problem_dir = File.join(GRADER_ROOT,'..',dir,old_problem_name)
                   new_problem_dir = File.join(GRADER_ROOT,'..',dir,new_problem_name)
                   if FileTest.exists? problem_dir
                     puts "Moving #{problem_dir} to #{new_problem_dir}."
                     File.rename(problem_dir, new_problem_dir)
                     tr_problem_dir = File.join(GRADER_ROOT,'..',dir,
                                                'test_request',old_problem_name)
                     new_tr_problem_dir = File.join(GRADER_ROOT,'..',dir,
                                                    'test_request',new_problem_name)
                     File.rename(tr_problem_dir, new_tr_problem_dir)
                   end
                 end
                 problem.name = new_problem_name
                 problem.save
               end
               def usage
                 puts <<USAGE
               Usage:
                 rename_problem [old_name] [new_name]
               or
                 rename_problem -f [filename]
               When using with -f, that file should contain, for each line, the old
               problem name and its new name.
               This script should be called at the judge root dir where dirs 'ev' and
               'ev-exam' are.
               USAGE
               end
               def valid_problem_name(name)
                 if name.length==0:
                     return false
                 else
-            -     return !(/^[a-zA-Z0-9_\-]+$/ === name)
+            +     return (/^[a-zA-Z0-9_\-]+$/ === name)
                 end
               end
               if (ARGV.length!=2)
                 usage
                 exit(0)
               end
               if ARGV[0]=='-f' and !FileTest.exists?(ARGV[1])
                 puts "File #{ARGV[1]} does not exist."
                 usage
                 exit(0)
               end
               # load grader environment
               GRADER_ENV = 'grading'
               require File.join(File.dirname(__FILE__),'config/environment')
               # boot rails, to be able to rename the problem
               RAILS_ENV = config.rails_env
               require RAILS_ROOT + '/config/environment'
               if ARGV[0]!='-f'
                 old_problem_name = ARGV[0]
                 new_problem_name = ARGV[1]
                 rename_problem(old_problem_name, new_problem_name)
               else
                 lines = IO.readlines(ARGV[1])
                 lines.each do |line|
                   items = line.split
                   if items.length==2
                     old_name, new_name = items
                     rename_problem(old_name, new_name)
                   end
                 end
               end

std-script/judge ¶ +6

               if ARGV.length < 2 || ARGV.length > 4
                 puts "Usage: judge <language> <program-source> [<test-result-directory>] [<sandbox-directory>]"
                 puts "  <sandbox-directory> is defaulted to ./sandbox"
                 puts "  <test-result-directory> is defaulted to ./test-result"
                 puts "WARNING: The judge script will forcefully create the (implicitly and explicitly) specified directories and remove anything inside it."
                 exit(127)
               end
               language = ARGV[0]
               if language != "c" && language != "c++" && language != "pas" && language != "java" && language != "ruby" && language != "python" && language != "php" && language != "haskell"
                 log "You specified a language that is not supported: #{language}."
                 exit(127)
               end
               source_file = ARGV[1]
               ENV['SOURCE_NAME'] = source_file
               if File.exist?(source_file) == false
                 log "The source file does not exist."
                 exit(127)
               end
               log "Making test result and sandbox directories..."
               current_dir = FileUtils.pwd
               current_dir.strip!
               if ARGV.length >= 3
                 test_result_dir = ARGV[2]
               else
                 test_result_dir = "#{current_dir}/test-result"
               end
               log "Test result directory: #{test_result_dir}"
               clear_and_create_empty_dir(test_result_dir)
               if ARGV.length >= 4
                 sandbox_dir = ARGV[3]
               else
                 sandbox_dir = "#{current_dir}/sandbox"
               end
               log "Sandbox directory: #{sandbox_dir}"
               clear_and_create_empty_dir(sandbox_dir)
               # Compile
               log
               log "Compiling..."
               call_and_log("Cannot copy the source file to #{sandbox_dir}") {
                 FileUtils.cp(source_file, sandbox_dir)
               }
               begin
                 Dir.chdir sandbox_dir
               rescue
                 log "ERROR: Cannot change directory to #{sandbox_dir}."
                 exit(127)
               end
               execute("#{problem_home}/script/compile #{language} #{source_file}", "Compilation error!")
               compile_message = open("compiler_message").read
               compile_message.strip!
               call_and_log("Cannot move the compiler message to #{test_result_dir}.") {
                 FileUtils.mv("compiler_message", test_result_dir)
               }
               if !FileTest.exist?("a.out")
                 log "Cannot compile the source code. See message in #{test_result_dir}/compile_message"
                 exit(127)
               else
                 call_and_log("Cannot move the compiled program to #{test_result_dir}") {
                   FileUtils.mv("a.out",test_result_dir)
                   if language == "java" then Dir["*.class"].each { |file| FileUtils.mv(file,test_result_dir)} end
                   if language == "python" then Dir["*.pyc"].each { |file| FileUtils.mv(file,test_result_dir)} end
                 }
                 FileUtils.rm_rf("#{sandbox_dir}/.")
               end
               require "#{problem_home}/script/test_dsl.rb"
               load "#{problem_home}/test_cases/all_tests.cfg"
               problem = Problem.get_instance
               if problem.well_formed? == false
                 log "The problem specification is not well formed."
                 exit(127)
               end
               # Doing the testing.
               (1..(problem.num_tests)).each do |test_num|
                 $stdout.print "[#{test_num}]"
                 $stdout.flush
                 log "Test number: #{test_num}"
                 call_and_log("Cannot copy the compiled program into #{sandbox_dir}") {
                   FileUtils.cp("#{test_result_dir}/a.out", sandbox_dir, :preserve => true)
                   if language == "java" then Dir["#{test_result_dir}/*.class"].each { |file| FileUtils.cp(file,sandbox_dir)} end
                   if language == "python" then Dir["#{test_result_dir}/*.pyc"].each { |file| FileUtils.cp(file,sandbox_dir)} end
                 }
+            +   #additionally copy any extra .txt file
+            +   data_files = Dir[problem_home + '/*.txt']
+            +   data_files.each do |file|
+            +     FileUtils.cp(file,sandbox_dir)
+            +   end
+            +
                 begin
                   execute("#{problem_home}/script/run #{language} #{test_num} ", "Error occured during execution of the run script")
                 rescue
                   # do nothing
                 end
                 call_and_log("Cannot create directory #{test_result_dir}/#{test_num}") {
                   FileUtils.mkdir "#{test_result_dir}/#{test_num}"
                 }
                 call_and_log("Cannot copy the result file into #{test_result_dir}/#{test_num}") {
                   FileUtils.mv "#{sandbox_dir}/result", "#{test_result_dir}/#{test_num}"
                 }
                 call_and_log("Cannot copy the comment file into #{test_result_dir}/#{test_num}") {
                   FileUtils.mv "#{sandbox_dir}/comment", "#{test_result_dir}/#{test_num}"
                 }
                 call_and_log("Cannot copy the output file into #{test_result_dir}/#{test_num}") {
                   FileUtils.mv "#{sandbox_dir}/output.txt", "#{test_result_dir}/#{test_num}"
                 }
                 call_and_log("Cannot clear #{sandbox_dir}") {
                   FileUtils.rm_rf(Dir.glob("#{sandbox_dir}/*"), :secure => true)
                 }
               end
               $stdout.print "[done]\n"
               # Grade
               log
               log "Grading..."
               begin
                 Dir.chdir test_result_dir
               rescue
                 log "ERROR: Cannot change directory to #{test_result_dir}."
                 exit(127)
               end
               execute("#{problem_home}/script/grade", "An error occured during grading!")
               log
               log "All done!"

templates/check.float ¶ +9

               #!/usr/bin/env ruby
               problem_home = ENV['PROBLEM_HOME']
               require "#{problem_home}/script/test_dsl.rb"
               if ARGV.length < 2
                 puts "Usage: check <language> <test-number> [<output-file>]"
                 exit(0)
               end
               language = ARGV[0]
               test_num = ARGV[1].to_i
               if ARGV.length >= 3
                 output_file_name = ARGV[2]
               else
                 output_file_name = "output.txt"
               end
               load "#{problem_home}/test_cases/all_tests.cfg"
               problem = Problem.get_instance
               output_file = File.new(output_file_name, "r")
               answer_file = File.new("#{problem_home}/test_cases/#{test_num}/answer-#{test_num}.txt")
               result_file = File.new("check_result", "w")
               output_file_content = output_file.read
               answer_file_content = answer_file.read
               report_correct = lambda {
                 result_file.write "Correct\n"
                 result_file.write problem.get_score(test_num)
                 result_file.write "\n"
                 result_file.close
                 exit(0)
               }
               report_wrong = lambda {
                 result_file.write "Incorrect\n"
                 result_file.write "0\n"
                 result_file.close
                 exit(0)
               }
               ##################
               # Your code here #
               ##################
               ########### THIS IS FOR CHECKING FLOAT with EPSILON error ##########
+            +
+            + def is_float?(fl)
+            +   !!Float(fl) rescue false
+            + end
+            +
               EPSILON = 0.000001
               out_items = output_file_content.split
               ans_items = answer_file_content.split
               if out_items.length != ans_items.length
                 report_wrong.call
               else
                 out_items.length.times do |i|
+            +     if is_float?(out_items[i]) && is_float?(ans_items[i])
                     out_value = out_items[i].to_f
                     ans_value = ans_items[i].to_f
                     if (out_value - ans_value).abs > EPSILON * [out_value.abs,ans_value.abs].max
                       report_wrong.call
                     end
+            +     else
+            +       report_wrong.call if out_items[i] != ans_items[i]
+            +     end
                 end
                 report_correct.call
               end

Write
Preview

You need to be logged in to leave comments. Login now

	Site-wide shortcuts
/	Use quick search box
g h	Goto home page
g g	Goto my private gists page
g G	Goto my public gists page
n r	New repository page
n g	New gist page

	Repositories
g s	Goto summary page
g c	Goto changelog page
g f	Goto files page
g F	Goto files page with file search activated
g p	Goto pull requests page
g o	Goto repository settings
g O	Goto repository permissions settings

Sign in to your account

Author

r241:19abb7c667e3 - Sat, 17 Jun 2017 06:19:08 - 7 files changed: 124 inserted, 4 deleted

Sign in to your account

Commit r241:19abb7c667e3 public

Author

r241:19abb7c667e3 - Sat, 17 Jun 2017 06:19:08 - 7 files changed: 124 inserted, 4 deleted

Commit `r241:19abb7c667e3` public