diff --git a/all_tests.cfg.erb b/all_tests.cfg.erb deleted file mode 100755 --- a/all_tests.cfg.erb +++ /dev/null @@ -1,13 +0,0 @@ -problem do - num_tests <%= num_testcases %> - full_score <%= num_testcases*10 %> - time_limit_each <%= options[:time_limit] %> - mem_limit_each <%= options[:mem_limit] %> - score_each 10 - -<% 1.upto(num_testcases) do |i| %> - run <%= i %> do - tests <%= i %> - end -<% end %> -end diff --git a/config/env_exam.rb.SAMPLE b/config/env_exam.rb.SAMPLE --- a/config/env_exam.rb.SAMPLE +++ b/config/env_exam.rb.SAMPLE @@ -1,20 +1,20 @@ - - +# +# See documentation in lib/configuration.rb +# Grader::Initializer.run do |config| - config.problems_dir = "/home/jittat/grader/ev" - config.user_result_dir = "/home/jittat/grader/result" + config.problems_dir = GRADER_ROOT + "/../ev" + config.user_result_dir = GRADER_ROOT + "/../result" config.talkative = true + config.logging = true + config.log_dir = GRADER_ROOT + "/../log" config.report_grader = true - - config.report_comment = lambda do |comment| - if comment.chomp =~ /^P+$/ # all P's - 'passed' - else - 'failed' - end - end - + + config.test_request_input_base_dir = RAILS_ROOT + "/data/test_request/input" + config.test_request_output_base_dir = RAILS_ROOT + "/data/test_request/output" + config.test_request_problem_templates_dir = config.problems_dir + "/test_request" + + config.comment_report_style = :short end diff --git a/config/env_grading.rb.SAMPLE b/config/env_grading.rb.SAMPLE --- a/config/env_grading.rb.SAMPLE +++ b/config/env_grading.rb.SAMPLE @@ -1,16 +1,19 @@ - - +# +# See documentation in lib/configuration.rb +# Grader::Initializer.run do |config| - - config.problems_dir = "/home/jittat/grader/ev" - config.user_result_dir = "/home/jittat/grader/result" + config.problems_dir = GRADER_ROOT + "/../ev" + config.user_result_dir = GRADER_ROOT + "/../result" config.talkative = true + config.logging = true + config.log_dir = GRADER_ROOT + "/../log" config.report_grader = true - config.report_comment = lambda do |comment| - comment.chomp - end - + config.test_request_input_base_dir = RAILS_ROOT + "/data/test_request/input" + config.test_request_output_base_dir = RAILS_ROOT + "/data/test_request/output" + config.test_request_problem_templates_dir = config.problems_dir + "/test_request" + + config.comment_report_style = :full end diff --git a/config/env_test.rb.SAMPLE b/config/env_test.rb.SAMPLE --- a/config/env_test.rb.SAMPLE +++ b/config/env_test.rb.SAMPLE @@ -1,25 +1,29 @@ - - +# +# See documentation in lib/configuration.rb +# Grader::Initializer.run do |config| + config.problems_dir = GRADER_ROOT + "/test/sandbox/ev" + config.user_result_dir = GRADER_ROOT + "/test/sandbox/result" - config.problems_dir = "/home/jittat/grader/scripts/test/sandbox/ev" - config.user_result_dir = "/home/jittat/grader/scripts/test/sandbox/result" - - config.talkative = true + config.talkative = false config.report_grader = false config.rails_env = 'test' - config.report_comment = lambda do |comment| - comment.chomp - end + config.comment_report_style = :full + config.test_request_input_base_dir = GRADER_ROOT + "/test/data/test_request/input" + config.test_request_output_base_dir = GRADER_ROOT + "/test/sandbox/test_request/output" + config.test_request_problem_templates_dir = GRADER_ROOT + "/test/data/test_request/problems" + + # + # These options are for testing + # class << config attr_accessor :test_data_dir, :test_sandbox_dir end - config.test_data_dir = "/home/jittat/grader/scripts/test/data" - config.test_sandbox_dir = "/home/jittat/grader/scripts/test/sandbox" - + config.test_data_dir = GRADER_ROOT + "/test/data" + config.test_sandbox_dir = GRADER_ROOT + "/test/sandbox" end diff --git a/config/environment.rb.SAMPLE b/config/environment.rb.SAMPLE --- a/config/environment.rb.SAMPLE +++ b/config/environment.rb.SAMPLE @@ -1,11 +1,10 @@ - # Rails app directory RAILS_ROOT = "/home/jittat/web_grader" GRADER_ROOT = "/home/jittat/grader/scripts" +# This load all required codes require File.join(File.dirname(__FILE__),'../lib/boot') # load the required environment file require File.dirname(__FILE__) + "/env_#{GRADER_ENV}.rb" - diff --git a/grader b/grader --- a/grader +++ b/grader @@ -16,16 +16,46 @@ Grader::Configuration.get_instance end -def talk(str) +def log_file_name + config.log_dir + + "/#{GRADER_ENV}_#{config.grader_mode}.#{Process.pid}" +end + +def log(str) if config.talkative puts str end + if config.logging + fp = File.open(log_file_name,"a") + fp.puts("GRADER: #{Time.new.strftime("%H:%M")} #{str}") + fp.close + end +end + +def display_manual + puts <= 1) and (ARGV[0]=='stop') stop_grader @@ -52,14 +82,20 @@ puts "environment: #{GRADER_ENV}" require File.join(File.dirname(__FILE__),'config/environment') -#reading rails environment -talk 'Reading rails environment' +# add grader_mode to config +# this is needed because method log needs it. TODO: clean this up +class << config + attr_accessor :grader_mode +end +config.grader_mode = grader_mode + +# reading rails environment +log 'Reading rails environment' RAILS_ENV = config.rails_env require RAILS_ROOT + '/config/environment' - -#register grader process +# register grader process if config.report_grader grader_proc = GraderProcess.register(config.grader_hostname, Process.pid, @@ -68,27 +104,45 @@ grader_proc = nil end -# create judge engine -engine = Grader::Engine.new(grader_proc) +#set loggin environment +ENV['GRADER_LOGGING'] = log_file_name + +# +# MAIN LOOP +# case grader_mode -when "queue" - talk 'Grader queue' +when "queue", "test_request" + log "Grader: #{grader_mode}" + if grader_mode=="queue" + engine = Grader::Engine.new + else + engine = Grader::Engine.new(Grader::TestRequestRoomMaker.new, + Grader::TestRequestReporter.new) + end + + runner = Grader::Runner.new(engine, grader_proc) while true if check_stopfile # created by calling grader stop clear_stopfile - puts "stopped" + log "stopped (with stop file)" break end - - task = engine.grade_oldest_task + + if grader_mode=="queue" + task = runner.grade_oldest_task + else + task = runner.grade_oldest_test_request + end if task==nil - sleep(5) + sleep(1) end end when "prob" + engine = Grader::Engine.new + runner = Grader::Runner.new(engine, grader_proc) grader_proc.report_active if grader_proc!=nil @@ -96,9 +150,12 @@ if prob==nil puts "cannot find problem: #{ARGV[2]}" else - engine.grade_problem(prob) + runner.grade_problem(prob) end - + +else + display_manual + exit(0) end # report inactive diff --git a/import_problem b/import_problem --- a/import_problem +++ b/import_problem @@ -5,6 +5,7 @@ # * copy testdata in the old format and create standard testcase config file require 'erb' +require 'fileutils' def input_filename(dir,i) "#{dir}/input-#{i}.txt" @@ -34,15 +35,18 @@ end end -GRADER_DIR = File.dirname(__FILE__) +SCRIPT_DIR = File.dirname(__FILE__) # print usage if ARGV.length < 3 - puts "using: import_problem prob_name importing_testcase_dir num_of_testcase [options] + puts < "user_id = #{u.id} and " + - "problem_id = #{prob.id}", - :order => 'submitted_at DESC') - if last_sub!=nil - grade(last_sub) - end + clear_script(copy_log,problem_home) + + rescue RuntimeError => msg + @reporter.report_error(submission,"Grading error: #{msg}") + + ensure + @room_maker.clean_up(submission) + Dir.chdir(current_dir) # this is really important end end - + protected def talk(str) @@ -83,66 +81,16 @@ end end - def save_source(submission,dir,fname) - f = File.open("#{dir}/#{fname}","w") - f.write(submission.source) - f.close - end - - def call_judge(problem_home,language,submission_out_dir,fname) + def call_judge(problem_home,language,grading_dir,fname) ENV['PROBLEM_HOME'] = problem_home - talk submission_out_dir - Dir.chdir submission_out_dir + talk grading_dir + Dir.chdir grading_dir cmd = "#{problem_home}/script/judge #{language} #{fname}" talk "CMD: #{cmd}" system(cmd) end - def read_result(test_result_dir) - cmp_msg_fname = "#{test_result_dir}/compiler_message" - cmp_file = File.open(cmp_msg_fname) - cmp_msg = cmp_file.read - cmp_file.close - - result_fname = "#{test_result_dir}/result" - comment_fname = "#{test_result_dir}/comment" - if FileTest.exist?(result_fname) - result_file = File.open(result_fname) - result = result_file.readline.to_i - result_file.close - - comment_file = File.open(comment_fname) - comment = comment_file.readline.chomp - comment_file.close - - return {:points => result, - :comment => comment, - :cmp_msg => cmp_msg} - else - return {:points => 0, - :comment => 'compile error', - :cmp_msg => cmp_msg} - end - end - - def save_result(submission,result) - problem = submission.problem - submission.graded_at = Time.now - points = result[:points] - submission.points = points - comment = @config.report_comment.call(result[:comment]) - if problem == nil - submission.grader_comment = 'PASSED: ' + comment + '(problem is nil)' - elsif points == problem.full_score - submission.grader_comment = 'PASSED: ' + comment - else - submission.grader_comment = 'FAILED: ' + comment - end - submission.compiler_message = result[:cmp_msg] - submission.save - end - def get_std_script_dir GRADER_ROOT + '/std-script' end diff --git a/lib/runner.rb b/lib/runner.rb new file mode 100644 --- /dev/null +++ b/lib/runner.rb @@ -0,0 +1,54 @@ +# +# A runner drives the engine into various tasks. +# + +module Grader + + class Runner + + def initialize(engine, grader_process=nil) + @engine = engine + @grader_process = grader_process + end + + def grade_oldest_task + task = Task.get_inqueue_and_change_status(Task::STATUS_GRADING) + if task!=nil + @grader_process.report_active(task) if @grader_process!=nil + + submission = Submission.find(task.submission_id) + @engine.grade(submission) + task.status_complete! + end + return task + end + + def grade_problem(problem) + users = User.find(:all) + users.each do |u| + puts "user: #{u.login}" + last_sub = Submission.find(:first, + :conditions => "user_id = #{u.id} and " + + "problem_id = #{prob.id}", + :order => 'submitted_at DESC') + if last_sub!=nil + @engine.grade(last_sub) + end + end + end + + def grade_oldest_test_request + test_request = TestRequest.get_inqueue_and_change_status(Task::STATUS_GRADING) + if test_request!=nil + @grader_process.report_active(test_request) if @grader_process!=nil + + @engine.grade(test_request) + test_request.status_complete! + end + return test_request + end + + end + +end + diff --git a/lib/submission_helper.rb b/lib/submission_helper.rb new file mode 100644 --- /dev/null +++ b/lib/submission_helper.rb @@ -0,0 +1,95 @@ +module Grader + + class SubmissionRoomMaker + def initialize + @config = Grader::Configuration.get_instance + end + + def produce_grading_room(submission) + user = submission.user + problem = submission.problem + grading_room = "#{@config.user_result_dir}/" + + "#{user.login}/#{problem.name}/#{submission.id}" + + FileUtils.mkdir_p(grading_room) + grading_room + end + + def find_problem_home(submission) + problem = submission.problem + "#{@config.problems_dir}/#{problem.name}" + end + + def save_source(submission,source_name) + dir = self.produce_grading_room(submission) + f = File.open("#{dir}/#{source_name}","w") + f.write(submission.source) + f.close + end + + def clean_up(submission) + end + end + + class SubmissionReporter + def initialize + @config = Grader::Configuration.get_instance + end + + def report(sub,test_result_dir) + save_result(sub,read_result(test_result_dir)) + end + + def report_error(sub,msg) + save_result(sub,{:points => 0, + :comment => "Grading error: #{msg}" }) + end + + protected + def read_result(test_result_dir) + cmp_msg_fname = "#{test_result_dir}/compiler_message" + cmp_file = File.open(cmp_msg_fname) + cmp_msg = cmp_file.read + cmp_file.close + + result_fname = "#{test_result_dir}/result" + comment_fname = "#{test_result_dir}/comment" + if FileTest.exist?(result_fname) + result_file = File.open(result_fname) + result = result_file.readline.to_i + result_file.close + + comment_file = File.open(comment_fname) + comment = comment_file.readline.chomp + comment_file.close + + return {:points => result, + :comment => comment, + :cmp_msg => cmp_msg} + else + return {:points => 0, + :comment => 'compile error', + :cmp_msg => cmp_msg} + end + end + + def save_result(submission,result) + problem = submission.problem + submission.graded_at = Time.now + points = result[:points] + submission.points = points + comment = @config.report_comment(result[:comment]) + if problem == nil + submission.grader_comment = 'PASSED: ' + comment + '(problem is nil)' + elsif points == problem.full_score + submission.grader_comment = 'PASSED: ' + comment + else + submission.grader_comment = 'FAILED: ' + comment + end + submission.compiler_message = result[:cmp_msg] or '' + submission.save + end + + end + +end diff --git a/lib/test_request_helper.rb b/lib/test_request_helper.rb new file mode 100644 --- /dev/null +++ b/lib/test_request_helper.rb @@ -0,0 +1,190 @@ +# +# This part contains various test_request helpers for interfacing +# with Grader::Engine. There are TestRequestRoomMaker and +# TestRequestReporter. + +module Grader + + # + # A TestRequestRoomMaker is a helper object for Engine + # - finds grading room: in user_result_dir/(user)/test_request/ ... + # - prepare problem configuration for grading --- basically it copy + # all config files, and copy user's input into the testcase + # directory. First, it finds the template from problem template + # directory; if it can't find a template, it'll use the template + # from default template. + class TestRequestRoomMaker + def initialize + @config = Grader::Configuration.get_instance + end + + def produce_grading_room(test_request) + grading_room = grading_room_dir(test_request) + FileUtils.mkdir_p(grading_room) + grading_room + end + + def find_problem_home(test_request) + problem_name = test_request.problem_name + + template_dir = "#{@config.test_request_problem_templates_dir}/" + problem_name + + raise "Test Request: error template not found" if !File.exists?(template_dir) + + problem_home = problem_home_dir(test_request) + FileUtils.mkdir_p(problem_home) + + copy_problem_template(template_dir,problem_home) + link_input_file(test_request,problem_home) + + problem_home + end + + def save_source(test_request,source_name) + dir = self.produce_grading_room(test_request) + submission = test_request.submission + f = File.open("#{dir}/#{source_name}","w") + f.write(submission.source) + f.close + end + + def clean_up(test_request) + problem_home = problem_home_dir(test_request) + remove_data_files(problem_home) + end + + protected + def grading_room_dir(test_request) + problem_name = test_request.problem_name + user = test_request.user + "#{@config.user_result_dir}" + + "/#{user.login}/test_request" + + "/#{problem_name}/#{test_request.id}" + end + + def problem_home_dir(test_request) + problem_name = test_request.problem_name + user = test_request.user + "#{@config.user_result_dir}" + + "/#{user.login}/test_request/#{problem_name}" + end + + def copy_problem_template(template_dir,problem_home) + cmd = "cp -R #{template_dir}/* #{problem_home}" + system_and_raise_when_fail(cmd,"Test Request: cannot copy problem template") + end + + def link_input_file(test_request,problem_home) + cmd = "ln -s #{test_request.input_file_name} #{problem_home}/test_cases/1/input-1.txt" + system_and_raise_when_fail(cmd,"Test Request: cannot link input file") + end + + def remove_data_files(problem_home) + if File.exists?("#{problem_home}/test_cases/1/input-1.txt") + cmd = "rm #{problem_home}/test_cases/1/*" + system_and_raise_when_fail(cmd,"Test Request: cannot remove data files") + end + end + + def system_and_raise_when_fail(cmd,msg) + if !system(cmd) + raise msg + end + end + + end + + class TestRequestReporter + def initialize + @config = Grader::Configuration.get_instance + end + + def report(test_request,test_result_dir) + save_result(test_request,read_result(test_result_dir)) + end + + def report_error(test_request, msg) + save_result(test_request, {:running_stat => "#{msg}"}) + end + + protected + def read_result(test_result_dir) + # TODO: + cmp_msg_fname = "#{test_result_dir}/compiler_message" + cmp_file = File.open(cmp_msg_fname) + cmp_msg = cmp_file.read + cmp_file.close + + result_file_name = "#{test_result_dir}/1/result" + + if File.exists?(result_file_name) + output_file_name = "#{test_result_dir}/1/output.txt" + results = File.open("#{test_result_dir}/1/result").readlines + stat = format_running_stat(results) + + return { + :output_file_name => output_file_name, + :running_stat => stat, + :comment => "", + :cmp_msg => cmp_msg} + else + return { + :running_stat => "", + :comment => "Compilation error", + :cmp_msg => cmp_msg} + end + end + + def format_running_stat(results) + running_time_line = results[-1] + + run_stat = "" + if !(/[Cc]orrect/.match(results[0])) + run_stat = results[0].chomp + end + + if res = /r(.*)u(.*)s/.match(running_time_line) + seconds = (res[1].to_f + res[2].to_f) + time_stat = "Time used: #{seconds} sec." + else + time_stat = "Time used: n/a sec." + end + return "#{run_stat}#{time_stat}" + end + + def save_result(test_request,result) + if result[:output_file_name]!=nil + test_request.output_file_name = link_output_file(test_request, + result[:output_file_name]) + end + test_request.graded_at = Time.now + test_request.compiler_message = (result[:cmp_msg] or '') + test_request.grader_comment = (result[:comment] or '') + test_request.running_stat = (result[:running_stat] or '') + test_request.save + end + + protected + def link_output_file(test_request, fname) + target_file_name = random_output_file_name(test_request.user, + test_request.problem) + FileUtils.mkdir_p(File.dirname(target_file_name)) + cmd = "ln -s #{fname} #{target_file_name}" + if !system(cmd) + raise "TestRequestReporter: cannot move output file" + end + return target_file_name + end + + def random_output_file_name(user,problem) + problem_name = TestRequest.name_of(problem) + begin + tmpname = "#{@config.test_request_output_base_dir}" + + "/#{user.login}/#{problem_name}/#{rand(10000)}" + end while File.exists?(tmpname) + tmpname + end + + end + +end diff --git a/std-script/grade b/std-script/grade --- a/std-script/grade +++ b/std-script/grade @@ -20,7 +20,7 @@ elsif comment =~ /[Tt]ime/ 'T' else - '?' + 'x' # these are run time errors end end diff --git a/std-script/run b/std-script/run --- a/std-script/run +++ b/std-script/run @@ -12,6 +12,18 @@ end end +def extract_time(t) + if (result=/^(.*)r(.*)u(.*)s/.match(t)) + {:real => result[1], :user => result[2], :sys => result[3]} + else + raise "Error reading running time" + end +end + +def compile_box(source,bin) + system("gcc #{source} -o #{bin}") +end + if ARGV.length < 2 || ARGV.length > 3 puts "Usage: run []" exit(127) @@ -55,8 +67,17 @@ # Copy the input file. #`cp #{problem_home}/test_cases/#{test_num}/#{input_file_name} .` +time_output_format = "%Er%Uu%Ss" + +# check if box is there, if not, compile it! +if !File.exists?("#{problem_home}/script/box") + log "WARNING: Compiling box: to increase efficiency, it should be compile manually" + compile_box("#{problem_home}/script/box.c", + "#{problem_home}/script/box") +end + # Run the program. -run_command = "/usr/bin/time -f \"%E\" 2>run_result #{problem_home}/script/box -a 2 -f -t #{time_limit} -m #{mem_limit} -i #{input_file_name} -o output.txt #{program_name}" +run_command = "/usr/bin/time -f \"#{time_output_format}\" 2>run_result #{problem_home}/script/box -a 2 -f -t #{time_limit} -m #{mem_limit} -i #{input_file_name} -o output.txt #{program_name}" log "Running test #{test_num}..." log run_command log @@ -71,6 +92,7 @@ run_result = run_result_file.readlines run_result_file.close time_elapsed = run_result[run_result.length-1] +running_time = extract_time(time_elapsed) report = lambda{ |status, points, comment| result_file.write status.strip @@ -99,6 +121,11 @@ report.call(run_result[0], 0, "No comment.\n") end +if running_time[:user].to_f + running_time[:sys].to_f > time_limit + log "Time limit exceeded." + report.call("Time limit exceeded", 0, "No comment.\n") +end + # Run 'check' to evaluate the output. #puts "There was no runtime error. Proceed to checking the output." check_command = "#{problem_home}/script/check #{language} #{test_num}" diff --git a/templates/all_tests.cfg.erb b/templates/all_tests.cfg.erb new file mode 100755 --- /dev/null +++ b/templates/all_tests.cfg.erb @@ -0,0 +1,13 @@ +problem do + num_tests <%= num_testcases %> + full_score <%= num_testcases*10 %> + time_limit_each <%= options[:time_limit] %> + mem_limit_each <%= options[:mem_limit] %> + score_each 10 + +<% 1.upto(num_testcases) do |i| %> + run <%= i %> do + tests <%= i %> + end +<% end %> +end diff --git a/templates/answer-1.txt b/templates/answer-1.txt new file mode 100644 diff --git a/templates/check_empty b/templates/check_empty new file mode 100755 --- /dev/null +++ b/templates/check_empty @@ -0,0 +1,47 @@ +#!/usr/bin/ruby + +problem_home = ENV['PROBLEM_HOME'] +require "#{problem_home}/script/test_dsl.rb" + +if ARGV.length < 2 + puts "Usage: check []" + exit(0) +end + +language = ARGV[0] +test_num = ARGV[1].to_i +if ARGV.length >= 3 + output_file_name = ARGV[2] +else + output_file_name = "output.txt" +end + +load "#{problem_home}/test_cases/all_tests.cfg" +problem = Problem.get_instance + +output_file = File.new(output_file_name, "r") +answer_file = File.new("#{problem_home}/test_cases/#{test_num}/answer-#{test_num}.txt") +result_file = File.new("check_result", "w") + +output_file_content = output_file.read +answer_file_content = answer_file.read + +report_correct = lambda { + result_file.write "Correct\n" + result_file.write problem.get_score(test_num) + result_file.write "\n" + result_file.close + exit(0) +} + +report_wrong = lambda { + result_file.write "Incorrect\n" + result_file.write "0\n" + result_file.close + exit(0) +} + +################## +# Your code here # +################## +report_correct.call diff --git a/templates/test_request_all_tests.cfg.erb b/templates/test_request_all_tests.cfg.erb new file mode 100755 --- /dev/null +++ b/templates/test_request_all_tests.cfg.erb @@ -0,0 +1,12 @@ +problem do + num_tests 1 + full_score 10 + time_limit_each <%= options[:time_limit] %> + mem_limit_each <%= options[:mem_limit] %> + score_each 10 + + run 1 do + tests 1 + end + +end diff --git a/test/data/add_nonzero_exit_status.c b/test/data/add_nonzero_exit_status.c new file mode 100644 --- /dev/null +++ b/test/data/add_nonzero_exit_status.c @@ -0,0 +1,10 @@ +#include + +int main() +{ + int a,b; + scanf("%d %d",&a,&b); + printf("%d\n",a+b); + return 10; +} + diff --git a/test/data/add_too_much_memory_dynamic.c b/test/data/add_too_much_memory_dynamic.c new file mode 100644 --- /dev/null +++ b/test/data/add_too_much_memory_dynamic.c @@ -0,0 +1,19 @@ +#include +#include + +int main() +{ + int a,b; + char *huge_array; + + scanf("%d %d",&a,&b); + + huge_array = (char *)malloc(5000000); + if(huge_array==NULL) + printf("NO!"); + else + printf("%d\n",a+b); + + return 0; +} + diff --git a/test/data/add_too_much_memory_static.c b/test/data/add_too_much_memory_static.c new file mode 100644 --- /dev/null +++ b/test/data/add_too_much_memory_static.c @@ -0,0 +1,12 @@ +#include + +int big_array[2000000]; + +int main() +{ + int a,b; + scanf("%d %d",&a,&b); + printf("%d\n",a+b); + return 0; +} + diff --git a/test/data/ev/test1/script/check b/test/data/ev/test1/script/check deleted file mode 100755 --- a/test/data/ev/test1/script/check +++ /dev/null @@ -1,59 +0,0 @@ -#!/usr/bin/ruby - -problem_home = ENV['PROBLEM_HOME'] -require "#{problem_home}/script/test_dsl.rb" - -if ARGV.length < 2 - puts "Usage: check []" - exit(0) -end - -language = ARGV[0] -test_num = ARGV[1].to_i -if ARGV.length >= 3 - output_file_name = ARGV[2] -else - output_file_name = "output.txt" -end - -load "#{problem_home}/test_cases/all_tests.cfg" -problem = Problem.get_instance - -output_file = File.new(output_file_name, "r") -answer_file = File.new("#{problem_home}/test_cases/#{test_num}/answer-#{test_num}.txt") -result_file = File.new("check_result", "w") - -output_file_content = output_file.read -answer_file_content = answer_file.read - -report_correct = lambda { - result_file.write "Correct\n" - result_file.write problem.get_score(test_num) - result_file.write "\n" - result_file.close - exit(0) -} - -report_wrong = lambda { - result_file.write "Incorrect\n" - result_file.write "0\n" - result_file.close - exit(0) -} - -################## -# Your code here # -################## -num_pattern = /^[0-9]*/ -if (output_file_content =~ num_pattern) == nil - report_wrong.call -end - -output_i = output_file_content.to_i -answer_i = answer_file_content.to_i - -if output_i == answer_i - report_correct.call -else - report_wrong.call -end diff --git a/test/data/ev/test1/test_cases/1/answer-1.txt b/test/data/ev/test1/test_cases/1/answer-1.txt deleted file mode 100644 --- a/test/data/ev/test1/test_cases/1/answer-1.txt +++ /dev/null @@ -1,1 +0,0 @@ -2 \ No newline at end of file diff --git a/test/data/ev/test1/test_cases/1/input-1.txt b/test/data/ev/test1/test_cases/1/input-1.txt deleted file mode 100644 --- a/test/data/ev/test1/test_cases/1/input-1.txt +++ /dev/null @@ -1,1 +0,0 @@ -1 1 \ No newline at end of file diff --git a/test/data/ev/test1/test_cases/10/answer-10.txt b/test/data/ev/test1/test_cases/10/answer-10.txt deleted file mode 100644 --- a/test/data/ev/test1/test_cases/10/answer-10.txt +++ /dev/null @@ -1,1 +0,0 @@ --256 \ No newline at end of file diff --git a/test/data/ev/test1/test_cases/10/input-10.txt b/test/data/ev/test1/test_cases/10/input-10.txt deleted file mode 100644 --- a/test/data/ev/test1/test_cases/10/input-10.txt +++ /dev/null @@ -1,1 +0,0 @@ --128 -128 \ No newline at end of file diff --git a/test/data/ev/test1/test_cases/2/answer-2.txt b/test/data/ev/test1/test_cases/2/answer-2.txt deleted file mode 100644 --- a/test/data/ev/test1/test_cases/2/answer-2.txt +++ /dev/null @@ -1,1 +0,0 @@ -32 \ No newline at end of file diff --git a/test/data/ev/test1/test_cases/2/input-2.txt b/test/data/ev/test1/test_cases/2/input-2.txt deleted file mode 100644 --- a/test/data/ev/test1/test_cases/2/input-2.txt +++ /dev/null @@ -1,1 +0,0 @@ -20 12 \ No newline at end of file diff --git a/test/data/ev/test1/test_cases/3/answer-3.txt b/test/data/ev/test1/test_cases/3/answer-3.txt deleted file mode 100644 --- a/test/data/ev/test1/test_cases/3/answer-3.txt +++ /dev/null @@ -1,1 +0,0 @@ -4 \ No newline at end of file diff --git a/test/data/ev/test1/test_cases/3/input-3.txt b/test/data/ev/test1/test_cases/3/input-3.txt deleted file mode 100644 --- a/test/data/ev/test1/test_cases/3/input-3.txt +++ /dev/null @@ -1,1 +0,0 @@ -1 3 \ No newline at end of file diff --git a/test/data/ev/test1/test_cases/3/test.cfg b/test/data/ev/test1/test_cases/3/test.cfg deleted file mode 100644 --- a/test/data/ev/test1/test_cases/3/test.cfg +++ /dev/null @@ -1,1 +0,0 @@ -mem_limit 32 \ No newline at end of file diff --git a/test/data/ev/test1/test_cases/4/answer-4.txt b/test/data/ev/test1/test_cases/4/answer-4.txt deleted file mode 100644 --- a/test/data/ev/test1/test_cases/4/answer-4.txt +++ /dev/null @@ -1,1 +0,0 @@ -64 \ No newline at end of file diff --git a/test/data/ev/test1/test_cases/4/input-4.txt b/test/data/ev/test1/test_cases/4/input-4.txt deleted file mode 100644 --- a/test/data/ev/test1/test_cases/4/input-4.txt +++ /dev/null @@ -1,1 +0,0 @@ -32 32 \ No newline at end of file diff --git a/test/data/ev/test1/test_cases/5/answer-5.txt b/test/data/ev/test1/test_cases/5/answer-5.txt deleted file mode 100644 --- a/test/data/ev/test1/test_cases/5/answer-5.txt +++ /dev/null @@ -1,1 +0,0 @@ --2 \ No newline at end of file diff --git a/test/data/ev/test1/test_cases/5/input-5.txt b/test/data/ev/test1/test_cases/5/input-5.txt deleted file mode 100644 --- a/test/data/ev/test1/test_cases/5/input-5.txt +++ /dev/null @@ -1,1 +0,0 @@ --1 -1 \ No newline at end of file diff --git a/test/data/ev/test1/test_cases/6/answer-6.txt b/test/data/ev/test1/test_cases/6/answer-6.txt deleted file mode 100644 --- a/test/data/ev/test1/test_cases/6/answer-6.txt +++ /dev/null @@ -1,1 +0,0 @@ --32 \ No newline at end of file diff --git a/test/data/ev/test1/test_cases/6/input-6.txt b/test/data/ev/test1/test_cases/6/input-6.txt deleted file mode 100644 --- a/test/data/ev/test1/test_cases/6/input-6.txt +++ /dev/null @@ -1,1 +0,0 @@ --16 -16 \ No newline at end of file diff --git a/test/data/ev/test1/test_cases/7/answer-7.txt b/test/data/ev/test1/test_cases/7/answer-7.txt deleted file mode 100644 --- a/test/data/ev/test1/test_cases/7/answer-7.txt +++ /dev/null @@ -1,1 +0,0 @@ -0 \ No newline at end of file diff --git a/test/data/ev/test1/test_cases/7/input-7.txt b/test/data/ev/test1/test_cases/7/input-7.txt deleted file mode 100644 --- a/test/data/ev/test1/test_cases/7/input-7.txt +++ /dev/null @@ -1,1 +0,0 @@ -0 0 \ No newline at end of file diff --git a/test/data/ev/test1/test_cases/8/answer-8.txt b/test/data/ev/test1/test_cases/8/answer-8.txt deleted file mode 100644 --- a/test/data/ev/test1/test_cases/8/answer-8.txt +++ /dev/null @@ -1,1 +0,0 @@ --1 \ No newline at end of file diff --git a/test/data/ev/test1/test_cases/8/input-8.txt b/test/data/ev/test1/test_cases/8/input-8.txt deleted file mode 100644 --- a/test/data/ev/test1/test_cases/8/input-8.txt +++ /dev/null @@ -1,1 +0,0 @@ -0 -1 \ No newline at end of file diff --git a/test/data/ev/test1/test_cases/8/test.cfg b/test/data/ev/test1/test_cases/8/test.cfg deleted file mode 100644 --- a/test/data/ev/test1/test_cases/8/test.cfg +++ /dev/null @@ -1,1 +0,0 @@ -mem_limit 64 \ No newline at end of file diff --git a/test/data/ev/test1/test_cases/9/answer-9.txt b/test/data/ev/test1/test_cases/9/answer-9.txt deleted file mode 100644 --- a/test/data/ev/test1/test_cases/9/answer-9.txt +++ /dev/null @@ -1,1 +0,0 @@ -256 \ No newline at end of file diff --git a/test/data/ev/test1/test_cases/9/input-9.txt b/test/data/ev/test1/test_cases/9/input-9.txt deleted file mode 100644 --- a/test/data/ev/test1/test_cases/9/input-9.txt +++ /dev/null @@ -1,1 +0,0 @@ -128 128 \ No newline at end of file diff --git a/test/data/ev/test1/test_cases/all_tests.cfg b/test/data/ev/test1/test_cases/all_tests.cfg deleted file mode 100644 --- a/test/data/ev/test1/test_cases/all_tests.cfg +++ /dev/null @@ -1,39 +0,0 @@ -problem do - num_tests 10 - full_score 135 - time_limit_each 1 - mem_limit_each 11 - score_each 10 - - run 1 do - tests 1, 2 - scores 10, 20 - time_limits 1, 2 - mem_limits 5, 6 - end - - run 2 do - tests 3, 4, 5, 6, 7 - score_each 10 - time_limit_each 3 - mem_limit_each 3 - end - - run 3 do - tests 8, 9, 10 - end - - test 8 do - score 30 - time_limit 3 - mem_limit 10 - end - - test 9 do - score 15 - end - - test 10 do - time_limit 1 - end -end diff --git a/test/data/ev/test2/script/check b/test/data/ev/test2/script/check deleted file mode 100755 --- a/test/data/ev/test2/script/check +++ /dev/null @@ -1,59 +0,0 @@ -#!/usr/bin/ruby - -problem_home = ENV['PROBLEM_HOME'] -require "#{problem_home}/script/test_dsl.rb" - -if ARGV.length < 2 - puts "Usage: check []" - exit(0) -end - -language = ARGV[0] -test_num = ARGV[1].to_i -if ARGV.length >= 3 - output_file_name = ARGV[2] -else - output_file_name = "output.txt" -end - -load "#{problem_home}/test_cases/all_tests.cfg" -problem = Problem.get_instance - -output_file = File.new(output_file_name, "r") -answer_file = File.new("#{problem_home}/test_cases/#{test_num}/answer-#{test_num}.txt") -result_file = File.new("check_result", "w") - -output_file_content = output_file.read -answer_file_content = answer_file.read - -report_correct = lambda { - result_file.write "Correct\n" - result_file.write problem.get_score(test_num) - result_file.write "\n" - result_file.close - exit(0) -} - -report_wrong = lambda { - result_file.write "Incorrect\n" - result_file.write "0\n" - result_file.close - exit(0) -} - -################## -# Your code here # -################## -num_pattern = /^[0-9]*/ -if (output_file_content =~ num_pattern) == nil - report_wrong.call -end - -output_i = output_file_content.to_i -answer_i = answer_file_content.to_i - -if output_i == answer_i - report_correct.call -else - report_wrong.call -end diff --git a/test/data/ev/test2/test_cases/1/answer-1.txt b/test/data/ev/test2/test_cases/1/answer-1.txt deleted file mode 100644 --- a/test/data/ev/test2/test_cases/1/answer-1.txt +++ /dev/null @@ -1,2 +0,0 @@ -2 - diff --git a/test/data/ev/test2/test_cases/1/input-1.txt b/test/data/ev/test2/test_cases/1/input-1.txt deleted file mode 100644 --- a/test/data/ev/test2/test_cases/1/input-1.txt +++ /dev/null @@ -1,2 +0,0 @@ -1 1 - diff --git a/test/data/ev/test2/test_cases/2/answer-2.txt b/test/data/ev/test2/test_cases/2/answer-2.txt deleted file mode 100644 --- a/test/data/ev/test2/test_cases/2/answer-2.txt +++ /dev/null @@ -1,2 +0,0 @@ -2 - diff --git a/test/data/ev/test2/test_cases/2/input-2.txt b/test/data/ev/test2/test_cases/2/input-2.txt deleted file mode 100644 --- a/test/data/ev/test2/test_cases/2/input-2.txt +++ /dev/null @@ -1,2 +0,0 @@ -1 1 - diff --git a/test/data/ev/test2/test_cases/all_tests.cfg b/test/data/ev/test2/test_cases/all_tests.cfg deleted file mode 100644 --- a/test/data/ev/test2/test_cases/all_tests.cfg +++ /dev/null @@ -1,20 +0,0 @@ -problem do - num_tests 2 - full_score 10 - time_limit_each 1 - mem_limit_each 16 - score_each 10 - - run 1 do - tests 1 - end - - test 2 do - time_limit 2 - end - - run 2 do - tests 2 - end - -end diff --git a/test/data/ev/test_memory/script/check b/test/data/ev/test_memory/script/check new file mode 100755 --- /dev/null +++ b/test/data/ev/test_memory/script/check @@ -0,0 +1,59 @@ +#!/usr/bin/ruby + +problem_home = ENV['PROBLEM_HOME'] +require "#{problem_home}/script/test_dsl.rb" + +if ARGV.length < 2 + puts "Usage: check []" + exit(0) +end + +language = ARGV[0] +test_num = ARGV[1].to_i +if ARGV.length >= 3 + output_file_name = ARGV[2] +else + output_file_name = "output.txt" +end + +load "#{problem_home}/test_cases/all_tests.cfg" +problem = Problem.get_instance + +output_file = File.new(output_file_name, "r") +answer_file = File.new("#{problem_home}/test_cases/#{test_num}/answer-#{test_num}.txt") +result_file = File.new("check_result", "w") + +output_file_content = output_file.read +answer_file_content = answer_file.read + +report_correct = lambda { + result_file.write "Correct\n" + result_file.write problem.get_score(test_num) + result_file.write "\n" + result_file.close + exit(0) +} + +report_wrong = lambda { + result_file.write "Incorrect\n" + result_file.write "0\n" + result_file.close + exit(0) +} + +################## +# Your code here # +################## +num_pattern = /^[0-9]*/ +if (output_file_content =~ num_pattern) == nil + report_wrong.call +end + +output_i = output_file_content.to_i +answer_i = answer_file_content.to_i + +if output_i == answer_i + report_correct.call +else + report_wrong.call +end diff --git a/test/data/ev/test_memory/test_cases/1/answer-1.txt b/test/data/ev/test_memory/test_cases/1/answer-1.txt new file mode 100644 --- /dev/null +++ b/test/data/ev/test_memory/test_cases/1/answer-1.txt @@ -0,0 +1,2 @@ +2 + diff --git a/test/data/ev/test_memory/test_cases/1/input-1.txt b/test/data/ev/test_memory/test_cases/1/input-1.txt new file mode 100644 --- /dev/null +++ b/test/data/ev/test_memory/test_cases/1/input-1.txt @@ -0,0 +1,2 @@ +1 1 + diff --git a/test/data/ev/test_memory/test_cases/2/answer-2.txt b/test/data/ev/test_memory/test_cases/2/answer-2.txt new file mode 100644 --- /dev/null +++ b/test/data/ev/test_memory/test_cases/2/answer-2.txt @@ -0,0 +1,2 @@ +2 + diff --git a/test/data/ev/test_memory/test_cases/2/input-2.txt b/test/data/ev/test_memory/test_cases/2/input-2.txt new file mode 100644 --- /dev/null +++ b/test/data/ev/test_memory/test_cases/2/input-2.txt @@ -0,0 +1,2 @@ +1 1 + diff --git a/test/data/ev/test_memory/test_cases/all_tests.cfg b/test/data/ev/test_memory/test_cases/all_tests.cfg new file mode 100644 --- /dev/null +++ b/test/data/ev/test_memory/test_cases/all_tests.cfg @@ -0,0 +1,20 @@ +problem do + num_tests 2 + full_score 20 + time_limit_each 1 + mem_limit_each 5 + score_each 10 + + run 1 do + tests 1 + end + + test 2 do + mem_limit 10 + end + + run 2 do + tests 2 + end + +end diff --git a/test/data/ev/test_normal/script/check b/test/data/ev/test_normal/script/check new file mode 100755 --- /dev/null +++ b/test/data/ev/test_normal/script/check @@ -0,0 +1,59 @@ +#!/usr/bin/ruby + +problem_home = ENV['PROBLEM_HOME'] +require "#{problem_home}/script/test_dsl.rb" + +if ARGV.length < 2 + puts "Usage: check []" + exit(0) +end + +language = ARGV[0] +test_num = ARGV[1].to_i +if ARGV.length >= 3 + output_file_name = ARGV[2] +else + output_file_name = "output.txt" +end + +load "#{problem_home}/test_cases/all_tests.cfg" +problem = Problem.get_instance + +output_file = File.new(output_file_name, "r") +answer_file = File.new("#{problem_home}/test_cases/#{test_num}/answer-#{test_num}.txt") +result_file = File.new("check_result", "w") + +output_file_content = output_file.read +answer_file_content = answer_file.read + +report_correct = lambda { + result_file.write "Correct\n" + result_file.write problem.get_score(test_num) + result_file.write "\n" + result_file.close + exit(0) +} + +report_wrong = lambda { + result_file.write "Incorrect\n" + result_file.write "0\n" + result_file.close + exit(0) +} + +################## +# Your code here # +################## +num_pattern = /^[0-9]*/ +if (output_file_content =~ num_pattern) == nil + report_wrong.call +end + +output_i = output_file_content.to_i +answer_i = answer_file_content.to_i + +if output_i == answer_i + report_correct.call +else + report_wrong.call +end diff --git a/test/data/ev/test_normal/test_cases/1/answer-1.txt b/test/data/ev/test_normal/test_cases/1/answer-1.txt new file mode 100644 --- /dev/null +++ b/test/data/ev/test_normal/test_cases/1/answer-1.txt @@ -0,0 +1,1 @@ +2 \ No newline at end of file diff --git a/test/data/ev/test_normal/test_cases/1/input-1.txt b/test/data/ev/test_normal/test_cases/1/input-1.txt new file mode 100644 --- /dev/null +++ b/test/data/ev/test_normal/test_cases/1/input-1.txt @@ -0,0 +1,1 @@ +1 1 \ No newline at end of file diff --git a/test/data/ev/test_normal/test_cases/10/answer-10.txt b/test/data/ev/test_normal/test_cases/10/answer-10.txt new file mode 100644 --- /dev/null +++ b/test/data/ev/test_normal/test_cases/10/answer-10.txt @@ -0,0 +1,1 @@ +-256 \ No newline at end of file diff --git a/test/data/ev/test_normal/test_cases/10/input-10.txt b/test/data/ev/test_normal/test_cases/10/input-10.txt new file mode 100644 --- /dev/null +++ b/test/data/ev/test_normal/test_cases/10/input-10.txt @@ -0,0 +1,1 @@ +-128 -128 \ No newline at end of file diff --git a/test/data/ev/test_normal/test_cases/2/answer-2.txt b/test/data/ev/test_normal/test_cases/2/answer-2.txt new file mode 100644 --- /dev/null +++ b/test/data/ev/test_normal/test_cases/2/answer-2.txt @@ -0,0 +1,1 @@ +32 \ No newline at end of file diff --git a/test/data/ev/test_normal/test_cases/2/input-2.txt b/test/data/ev/test_normal/test_cases/2/input-2.txt new file mode 100644 --- /dev/null +++ b/test/data/ev/test_normal/test_cases/2/input-2.txt @@ -0,0 +1,1 @@ +20 12 \ No newline at end of file diff --git a/test/data/ev/test_normal/test_cases/3/answer-3.txt b/test/data/ev/test_normal/test_cases/3/answer-3.txt new file mode 100644 --- /dev/null +++ b/test/data/ev/test_normal/test_cases/3/answer-3.txt @@ -0,0 +1,1 @@ +4 \ No newline at end of file diff --git a/test/data/ev/test_normal/test_cases/3/input-3.txt b/test/data/ev/test_normal/test_cases/3/input-3.txt new file mode 100644 --- /dev/null +++ b/test/data/ev/test_normal/test_cases/3/input-3.txt @@ -0,0 +1,1 @@ +1 3 \ No newline at end of file diff --git a/test/data/ev/test_normal/test_cases/3/test.cfg b/test/data/ev/test_normal/test_cases/3/test.cfg new file mode 100644 --- /dev/null +++ b/test/data/ev/test_normal/test_cases/3/test.cfg @@ -0,0 +1,1 @@ +mem_limit 32 \ No newline at end of file diff --git a/test/data/ev/test_normal/test_cases/4/answer-4.txt b/test/data/ev/test_normal/test_cases/4/answer-4.txt new file mode 100644 --- /dev/null +++ b/test/data/ev/test_normal/test_cases/4/answer-4.txt @@ -0,0 +1,1 @@ +64 \ No newline at end of file diff --git a/test/data/ev/test_normal/test_cases/4/input-4.txt b/test/data/ev/test_normal/test_cases/4/input-4.txt new file mode 100644 --- /dev/null +++ b/test/data/ev/test_normal/test_cases/4/input-4.txt @@ -0,0 +1,1 @@ +32 32 \ No newline at end of file diff --git a/test/data/ev/test_normal/test_cases/5/answer-5.txt b/test/data/ev/test_normal/test_cases/5/answer-5.txt new file mode 100644 --- /dev/null +++ b/test/data/ev/test_normal/test_cases/5/answer-5.txt @@ -0,0 +1,1 @@ +-2 \ No newline at end of file diff --git a/test/data/ev/test_normal/test_cases/5/input-5.txt b/test/data/ev/test_normal/test_cases/5/input-5.txt new file mode 100644 --- /dev/null +++ b/test/data/ev/test_normal/test_cases/5/input-5.txt @@ -0,0 +1,1 @@ +-1 -1 \ No newline at end of file diff --git a/test/data/ev/test_normal/test_cases/6/answer-6.txt b/test/data/ev/test_normal/test_cases/6/answer-6.txt new file mode 100644 --- /dev/null +++ b/test/data/ev/test_normal/test_cases/6/answer-6.txt @@ -0,0 +1,1 @@ +-32 \ No newline at end of file diff --git a/test/data/ev/test_normal/test_cases/6/input-6.txt b/test/data/ev/test_normal/test_cases/6/input-6.txt new file mode 100644 --- /dev/null +++ b/test/data/ev/test_normal/test_cases/6/input-6.txt @@ -0,0 +1,1 @@ +-16 -16 \ No newline at end of file diff --git a/test/data/ev/test_normal/test_cases/7/answer-7.txt b/test/data/ev/test_normal/test_cases/7/answer-7.txt new file mode 100644 --- /dev/null +++ b/test/data/ev/test_normal/test_cases/7/answer-7.txt @@ -0,0 +1,1 @@ +0 \ No newline at end of file diff --git a/test/data/ev/test_normal/test_cases/7/input-7.txt b/test/data/ev/test_normal/test_cases/7/input-7.txt new file mode 100644 --- /dev/null +++ b/test/data/ev/test_normal/test_cases/7/input-7.txt @@ -0,0 +1,1 @@ +0 0 \ No newline at end of file diff --git a/test/data/ev/test_normal/test_cases/8/answer-8.txt b/test/data/ev/test_normal/test_cases/8/answer-8.txt new file mode 100644 --- /dev/null +++ b/test/data/ev/test_normal/test_cases/8/answer-8.txt @@ -0,0 +1,1 @@ +-1 \ No newline at end of file diff --git a/test/data/ev/test_normal/test_cases/8/input-8.txt b/test/data/ev/test_normal/test_cases/8/input-8.txt new file mode 100644 --- /dev/null +++ b/test/data/ev/test_normal/test_cases/8/input-8.txt @@ -0,0 +1,1 @@ +0 -1 \ No newline at end of file diff --git a/test/data/ev/test_normal/test_cases/8/test.cfg b/test/data/ev/test_normal/test_cases/8/test.cfg new file mode 100644 --- /dev/null +++ b/test/data/ev/test_normal/test_cases/8/test.cfg @@ -0,0 +1,1 @@ +mem_limit 64 \ No newline at end of file diff --git a/test/data/ev/test_normal/test_cases/9/answer-9.txt b/test/data/ev/test_normal/test_cases/9/answer-9.txt new file mode 100644 --- /dev/null +++ b/test/data/ev/test_normal/test_cases/9/answer-9.txt @@ -0,0 +1,1 @@ +256 \ No newline at end of file diff --git a/test/data/ev/test_normal/test_cases/9/input-9.txt b/test/data/ev/test_normal/test_cases/9/input-9.txt new file mode 100644 --- /dev/null +++ b/test/data/ev/test_normal/test_cases/9/input-9.txt @@ -0,0 +1,1 @@ +128 128 \ No newline at end of file diff --git a/test/data/ev/test_normal/test_cases/all_tests.cfg b/test/data/ev/test_normal/test_cases/all_tests.cfg new file mode 100644 --- /dev/null +++ b/test/data/ev/test_normal/test_cases/all_tests.cfg @@ -0,0 +1,39 @@ +problem do + num_tests 10 + full_score 135 + time_limit_each 1 + mem_limit_each 11 + score_each 10 + + run 1 do + tests 1, 2 + scores 10, 20 + time_limits 1, 2 + mem_limits 5, 6 + end + + run 2 do + tests 3, 4, 5, 6, 7 + score_each 10 + time_limit_each 3 + mem_limit_each 3 + end + + run 3 do + tests 8, 9, 10 + end + + test 8 do + score 30 + time_limit 3 + mem_limit 10 + end + + test 9 do + score 15 + end + + test 10 do + time_limit 1 + end +end diff --git a/test/data/ev/test_timeout/script/check b/test/data/ev/test_timeout/script/check new file mode 100755 --- /dev/null +++ b/test/data/ev/test_timeout/script/check @@ -0,0 +1,59 @@ +#!/usr/bin/ruby + +problem_home = ENV['PROBLEM_HOME'] +require "#{problem_home}/script/test_dsl.rb" + +if ARGV.length < 2 + puts "Usage: check []" + exit(0) +end + +language = ARGV[0] +test_num = ARGV[1].to_i +if ARGV.length >= 3 + output_file_name = ARGV[2] +else + output_file_name = "output.txt" +end + +load "#{problem_home}/test_cases/all_tests.cfg" +problem = Problem.get_instance + +output_file = File.new(output_file_name, "r") +answer_file = File.new("#{problem_home}/test_cases/#{test_num}/answer-#{test_num}.txt") +result_file = File.new("check_result", "w") + +output_file_content = output_file.read +answer_file_content = answer_file.read + +report_correct = lambda { + result_file.write "Correct\n" + result_file.write problem.get_score(test_num) + result_file.write "\n" + result_file.close + exit(0) +} + +report_wrong = lambda { + result_file.write "Incorrect\n" + result_file.write "0\n" + result_file.close + exit(0) +} + +################## +# Your code here # +################## +num_pattern = /^[0-9]*/ +if (output_file_content =~ num_pattern) == nil + report_wrong.call +end + +output_i = output_file_content.to_i +answer_i = answer_file_content.to_i + +if output_i == answer_i + report_correct.call +else + report_wrong.call +end diff --git a/test/data/ev/test_timeout/test_cases/1/answer-1.txt b/test/data/ev/test_timeout/test_cases/1/answer-1.txt new file mode 100644 --- /dev/null +++ b/test/data/ev/test_timeout/test_cases/1/answer-1.txt @@ -0,0 +1,2 @@ +2 + diff --git a/test/data/ev/test_timeout/test_cases/1/input-1.txt b/test/data/ev/test_timeout/test_cases/1/input-1.txt new file mode 100644 --- /dev/null +++ b/test/data/ev/test_timeout/test_cases/1/input-1.txt @@ -0,0 +1,2 @@ +1 1 + diff --git a/test/data/ev/test_timeout/test_cases/2/answer-2.txt b/test/data/ev/test_timeout/test_cases/2/answer-2.txt new file mode 100644 --- /dev/null +++ b/test/data/ev/test_timeout/test_cases/2/answer-2.txt @@ -0,0 +1,2 @@ +2 + diff --git a/test/data/ev/test_timeout/test_cases/2/input-2.txt b/test/data/ev/test_timeout/test_cases/2/input-2.txt new file mode 100644 --- /dev/null +++ b/test/data/ev/test_timeout/test_cases/2/input-2.txt @@ -0,0 +1,2 @@ +1 1 + diff --git a/test/data/ev/test_timeout/test_cases/all_tests.cfg b/test/data/ev/test_timeout/test_cases/all_tests.cfg new file mode 100644 --- /dev/null +++ b/test/data/ev/test_timeout/test_cases/all_tests.cfg @@ -0,0 +1,20 @@ +problem do + num_tests 2 + full_score 20 + time_limit_each 1 + mem_limit_each 16 + score_each 10 + + run 1 do + tests 1 + end + + test 2 do + time_limit 2 + end + + run 2 do + tests 2 + end + +end diff --git a/test/data/test2_1-5sec.c b/test/data/test2_1-5sec.c --- a/test/data/test2_1-5sec.c +++ b/test/data/test2_1-5sec.c @@ -1,4 +1,5 @@ #include +#include int main() { @@ -15,5 +16,6 @@ b+=c; } } + exit(0); } diff --git a/test/data/test_request/input/in1.txt b/test/data/test_request/input/in1.txt new file mode 100644 --- /dev/null +++ b/test/data/test_request/input/in1.txt @@ -0,0 +1,2 @@ +10 +20 diff --git a/test/data/test_request/problems/test_normal/script/check b/test/data/test_request/problems/test_normal/script/check new file mode 100755 --- /dev/null +++ b/test/data/test_request/problems/test_normal/script/check @@ -0,0 +1,47 @@ +#!/usr/bin/ruby + +problem_home = ENV['PROBLEM_HOME'] +require "#{problem_home}/script/test_dsl.rb" + +if ARGV.length < 2 + puts "Usage: check []" + exit(0) +end + +language = ARGV[0] +test_num = ARGV[1].to_i +if ARGV.length >= 3 + output_file_name = ARGV[2] +else + output_file_name = "output.txt" +end + +load "#{problem_home}/test_cases/all_tests.cfg" +problem = Problem.get_instance + +output_file = File.new(output_file_name, "r") +answer_file = File.new("#{problem_home}/test_cases/#{test_num}/answer-#{test_num}.txt") +result_file = File.new("check_result", "w") + +output_file_content = output_file.read +answer_file_content = answer_file.read + +report_correct = lambda { + result_file.write "Correct\n" + result_file.write problem.get_score(test_num) + result_file.write "\n" + result_file.close + exit(0) +} + +report_wrong = lambda { + result_file.write "Incorrect\n" + result_file.write "0\n" + result_file.close + exit(0) +} + +################## +# Your code here # +################## +report_correct.call diff --git a/test/data/test_request/problems/test_normal/test_cases/1/answer-1.txt b/test/data/test_request/problems/test_normal/test_cases/1/answer-1.txt new file mode 100644 diff --git a/test/data/test_request/problems/test_normal/test_cases/all_tests.cfg b/test/data/test_request/problems/test_normal/test_cases/all_tests.cfg new file mode 100644 --- /dev/null +++ b/test/data/test_request/problems/test_normal/test_cases/all_tests.cfg @@ -0,0 +1,11 @@ +problem do + num_tests 1 + full_score 10 + time_limit_each 1 + mem_limit_each 16 + score_each 10 + + run 1 do + tests 1 + end +end diff --git a/test/engine_spec.rb b/test/engine_spec.rb new file mode 100644 --- /dev/null +++ b/test/engine_spec.rb @@ -0,0 +1,246 @@ +require File.join(File.dirname(__FILE__),'spec_helper') +require File.join(File.dirname(__FILE__),'engine_spec_helper') + +describe "A grader engine, when grading submissions" do + + include GraderEngineHelperMethods + + before(:each) do + @config = Grader::Configuration.get_instance + + # this test is from Pong + @problem_test_normal = stub(Problem, + :id => 1, :name => 'test_normal', + :full_score => 135) + @user_user1 = stub(User, + :id => 1, :login => 'user1') + + @engine = Grader::Engine.new + init_sandbox + end + + it "should grade normal submission" do + grader_should(:grade => "test1_correct.c", + :on => @problem_test_normal, + :and_report => { + :score => 135, + :comment => /^PASSED/}) + end + + + it "should produce error message when submission cannot compile" do + grader_should(:grade => "test1_compile_error.c", + :on => @problem_test_normal, + :and_report => { + :score => 0, + :comment => 'FAILED: compile error', + :compiler_message => /[Ee]rror/}) + end + + it "should produce timeout error when submission runs forever" do + @problem_test_timeout = stub(Problem, + :id => 1, :name => 'test_timeout', + :full_score => 10) + grader_should(:grade => "test2_timeout.c", + :on => @problem_test_timeout, + :and_report => { + :score => 0, + :comment => 'FAILED: TT'}) + end + + it "should produce timeout error correctly when submission runs slower than expected in less than a second" do + @problem_test_timeout = stub(Problem, + :id => 1, :name => 'test_timeout', + :full_score => 20) + grader_should(:grade => "test2_1-5sec.c", + :on => @problem_test_timeout, + :and_report => { + :score => 10, + :comment => 'FAILED: TP'}) + end + + it "should produce runtime error when submission uses too much static memory" do + @problem_test_memory = stub(Problem, + :id => 1, :name => 'test_memory', + :full_score => 20) + grader_should(:grade => "add_too_much_memory_static.c", + :on => @problem_test_memory, + :and_report => { + :score => 10, + :comment => /FAILED: [^P]P/}) + end + + it "should not allow submission to allocate too much dynamic memory" do + @problem_test_memory = stub(Problem, + :id => 1, :name => 'test_memory', + :full_score => 20) + grader_should(:grade => "add_too_much_memory_dynamic.c", + :on => @problem_test_memory, + :and_report => { + :score => 10, + :comment => /FAILED: [^P]P/}) + end + + it "should fail submission with non-zero exit status" do + violated("has not been implemented") + end + + def grader_should(args) + @user1 = stub(User, + :id => 1, :login => 'user1') + submission = + create_submission_from_file(1, @user1, args[:on], args[:grade]) + submission.should_receive(:graded_at=) + + expected_score = args[:and_report][:score] + expected_comment = args[:and_report][:comment] + if args[:and_report][:compiler_message]!=nil + expected_compiler_message = args[:and_report][:compiler_message] + else + expected_compiler_message = '' + end + + submission.should_receive(:points=).with(expected_score) + submission.should_receive(:grader_comment=).with(expected_comment) + submission.should_receive(:compiler_message=).with(expected_compiler_message) + submission.should_receive(:save) + + @engine.grade(submission) + end + + protected + + def create_normal_submission_mock_from_file(source_fname) + create_submission_from_file(1, @user_user1, @problem_test_normal, source_fname) + end + +end + +describe "A grader engine, when grading test requests" do + + include GraderEngineHelperMethods + + before(:each) do + @config = Grader::Configuration.get_instance + @engine = Grader::Engine.new(Grader::TestRequestRoomMaker.new, + Grader::TestRequestReporter.new) + init_sandbox + end + + it "should report error if there is no problem template" do + problem = stub(Problem, + :id => 1, :name => 'nothing') + grader_should(:grade => 'test1_correct.c', + :on => problem, + :with => 'in1.txt', + :and_report => { + :graded_at= => nil, + :compiler_message= => '', + :grader_comment= => '', + :running_stat= => /template not found/, + :save => nil}) + end + + it "should run test request and produce output file" do + problem = stub(Problem, + :id => 1, :name => 'test_normal') + grader_should(:grade => 'test1_correct.c', + :on => problem, + :with => 'in1.txt', + :and_report => { + :graded_at= => nil, + :compiler_message= => '', + :grader_comment= => '', + :running_stat= => /0.0 sec./, + :output_file_name= => lambda { |fname| + File.exists?(fname).should be_true + }, + :save => nil}) + end + + it "should clean up problem directory after running test request" do + problem = stub(Problem, + :id => 1, :name => 'test_normal') + grader_should(:grade => 'test1_correct.c', + :on => problem, + :with => 'in1.txt', + :and_report => { + :graded_at= => nil, + :compiler_message= => '', + :grader_comment= => '', + :running_stat= => nil, + :output_file_name= => nil, + :save => nil}) + File.exists?(@config.user_result_dir + "/test_request/test_normal/test_cases/1/input-1.txt").should be_false + end + + it "should compile test request with error and report compilation error" do + problem = stub(Problem, + :id => 1, :name => 'test_normal') + grader_should(:grade => 'test1_compile_error.c', + :on => problem, + :with => 'in1.txt', + :and_report => { + :graded_at= => nil, + :compiler_message= => /.+/, + :grader_comment= => /[Cc]ompil.*error/, + :running_stat= => '', + :save => nil}) + end + + it "should report exit status" do + problem = stub(Problem, + :id => 1, :name => 'test_normal') + grader_should(:grade => 'add_nonzero_exit_status.c', + :on => problem, + :with => 'in1.txt', + :and_report => { + :graded_at= => nil, + :compiler_message= => '', + :grader_comment= => '', + :running_stat= => /[Ee]xit.*status.*10.*0.0 sec./, + :output_file_name= => lambda { |fname| + File.exists?(fname).should be_true + }, + :save => nil}) + end + + protected + def grader_should(args) + @user1 = stub(User, + :id => 1, :login => 'user1') + + problem = args[:on] + input_file = @config.test_request_input_base_dir + "/" + args[:with] + + submission = + create_submission_from_file(1, @user1, args[:on], args[:grade]) + + test_request = stub(TestRequest, + :id => 1, + :user => @user1, + :problem => problem, + :submission => submission, + :input_file_name => input_file, + :language => submission.language, + :problem_name => problem.name) + + expectations = args[:and_report] + + expectations.each do |key,val| + if val==nil + test_request.should_receive(key) + elsif val.class == Proc + test_request.should_receive(key) { |fname| + val.call(fname) + } + else + test_request.should_receive(key).with(val) + end + end + + @engine.grade(test_request) + end + +end + diff --git a/test/engine_spec_helper.rb b/test/engine_spec_helper.rb new file mode 100644 --- /dev/null +++ b/test/engine_spec_helper.rb @@ -0,0 +1,30 @@ +module GraderEngineHelperMethods + + def clear_sandbox + config = Grader::Configuration.get_instance + clear_cmd = "rm -rf #{config.test_sandbox_dir}/*" + system(clear_cmd) + end + + def init_sandbox + config = Grader::Configuration.get_instance + clear_sandbox + Dir.mkdir config.user_result_dir + cp_cmd = "cp -R #{config.test_data_dir}/ev #{config.test_sandbox_dir}" + system(cp_cmd) + end + + def create_submission_from_file(id, user, problem, + source_fname, language=nil) + + language = stub(Language, :name => 'c', :ext => 'c') if language==nil + + config = Grader::Configuration.get_instance + source = File.open(config.test_data_dir + "/" + source_fname).read + stub(Submission, + :id => id, :user => user, :problem => problem, + :source => source, :language => language) + end + +end + diff --git a/test/engine_test.rb b/test/engine_test.rb --- a/test/engine_test.rb +++ b/test/engine_test.rb @@ -76,14 +76,14 @@ end def test_timeout_submission_running_one_and_a_half_second - @problem_test2 = stub(:id => 1, :name => 'test2', :full_score => 10) + @problem_test2 = stub(:id => 1, :name => 'test2', :full_score => 20) @user_user1 = stub(:id => 1, :login => 'user1') submission = create_submission_from_file(1, @user_user1, @problem_test2, "test2_1-5sec.c") submission.expects(:graded_at=) - submission.expects(:points=).with(0) + submission.expects(:points=).with(10) submission.expects(:grader_comment=).with do |value| /^FAILED: TP$/.match value end diff --git a/test/runner_spec.rb b/test/runner_spec.rb new file mode 100644 --- /dev/null +++ b/test/runner_spec.rb @@ -0,0 +1,65 @@ +require File.join(File.dirname(__FILE__),'spec_helper') +require File.join(File.dirname(__FILE__),'engine_spec_helper') + +describe "A grader runner, when grade task" do + + include GraderEngineHelperMethods + + before(:each) do + @config = Grader::Configuration.get_instance + @problem_test_normal = stub(Problem, + :id => 1, :name => 'test_normal', + :full_score => 135) + @user_user1 = stub(User, + :id => 1, :login => 'user1') + + @engine = Grader::Engine.new + @runner = Grader::Runner.new(@engine) + init_sandbox + end + + it "should just return nil when there is no submission" do + Task.should_receive(:get_inqueue_and_change_status).and_return(nil) + @runner.grade_oldest_task.should be_nil + end + + it "should grade oldest task in queue" do + submission = create_normal_submission_mock_from_file("test1_correct.c") + + submission.should_receive(:graded_at=) + submission.should_receive(:points=).with(135) + submission.should_receive(:grader_comment=).with(/^PASSED/) + submission.should_receive(:compiler_message=).with('') + submission.should_receive(:save) + + # mock task + task = stub(Task,:id => 1, :submission_id => submission.id) + Task.should_receive(:get_inqueue_and_change_status).and_return(task) + task.should_receive(:status_complete!) + + # mock Submission + Submission.should_receive(:find). + with(task.submission_id). + and_return(submission) + + @runner.grade_oldest_task + end + + # to be converted + def test_grade_oldest_task_with_grader_process + grader_process = stub + grader_process.expects(:report_active) + + @runner = Grader::Runner.new(@engine,grader_process) + + test_grade_oldest_task + end + + protected + + def create_normal_submission_mock_from_file(source_fname) + create_submission_from_file(1, @user_user1, @problem_test_normal, source_fname) + end + +end + diff --git a/test/spec_helper.rb b/test/spec_helper.rb new file mode 100644 --- /dev/null +++ b/test/spec_helper.rb @@ -0,0 +1,25 @@ + +# This test helper loads the grader's environment and rails environment + +GRADER_ENV = 'test' +require File.join(File.dirname(__FILE__),'../config/environment') + + +# this shall be removed soon +RAILS_ENV = Grader::Configuration.get_instance.rails_env +require RAILS_ROOT + '/config/environment' + +# make sure not to access real database! +# taken from http://blog.jayfields.com/2006/06/ruby-on-rails-unit-tests.html + +class UnitTest + def self.TestCase + class << ActiveRecord::Base + def connection + raise 'You cannot access the database from a unit test' +# raise InvalidActionError, 'You cannot access the database from a unit test', caller + end + end + Test::Unit::TestCase + end +end