cafe-grader-judge-scripts Commit

cafe-grader-judge-scripts

Commit `r206:cf0534046f00` public

parent | child

Description:

Merge pull request #10 from wytesk133/master Fixed some Ruby scripts

Commit status:

[Not Reviewed]

References:

merge default

Diff options:

| | | |

Comments:

0 Commit comments 0 Inline Comments

Unresolved TODOs:

There are no unresolved TODOs

Add another comment

r206:cf0534046f00 - Thu, 01 Jan 2015 02:08:47 - 8 files changed: 11 inserted, 11 deleted

new_problem ¶ +1 -1

-            - #!/usr/bin/ruby
+            + #!/usr/bin/env ruby
               # new_problem:
               #  * creates a directory for a problem in the current directory,
               #  * create standard testcase config file
               require 'erb'
               def process_options(options)
                 i = 2
                 while i<ARGV.length
                   if ARGV[i]=='-t'
                     options[:time_limit] = ARGV[i+1].to_i if ARGV.length>i+1
                     i += 1
                   end
                   if ARGV[i]=='-m'
                     options[:mem_limit] = ARGV[i+1].to_i if ARGV.length>i+1
                     i += 1
                   end
                   i += 1
                 end
               end
               puts "This script is out of dated, shall be fixed soon"
               puts "Right now, you can create raw_ev and import"
               exit(0)
               GRADER_DIR = File.dirname(__FILE__)
               # print usage
               if ARGV.length < 2
                 puts <<USAGE
               using: new_problem problem number_of_testcase [options]
               * creates a directory for a problem in the current directory,
               * create standard testcase config file
               * options: -t time-limit (in seconds)
                          -m mem-limit (in MB)
               USAGE
                 exit(127)
               end
               # processing arguments
               problem = ARGV[0]
               num_testcases = ARGV[1].to_i
               options = {:time_limit => 1,  :mem_limit => 16}
               process_options(options)
               # start working
               puts "creating directories"
               system("mkdir #{problem}")
               system("mkdir #{problem}/script")
               system("mkdir #{problem}/test_cases")
               puts "creating testcases directories"
 .upto(num_testcases) do |i|
                 system("mkdir #{problem}/test_cases/#{i}")
               end
               # generating all_tests.cfg
               puts "generating testcase config file"
               template = File.open(File.dirname(__FILE__) + "/templates/all_tests.cfg.erb").read
               all_test_cfg = ERB.new(template)
               cfg_file = File.open("#{problem}/test_cases/all_tests.cfg","w")
               cfg_file.puts all_test_cfg.result
               cfg_file.close
               puts "done"

templates/check_wrapper ¶ +1 -1

-            - #!/usr/bin/ruby
+            + #!/usr/bin/env ruby
               #
               # This is a check script wrapper.  It read all required information
               # and call a real check script call REAL_CHECK_SCRIPT in directory
               # [problem_home]/script
               #
               REAL_CHECK_SCRIPT = "<%= script_name %>"
               # The REAL_CHECK_SCRIPT is called with:
               #
               #  (script) <lang> <test-num> <in-file> <out-file> <ans-file> <full-score>
               #
               # and REAL_CHECK_SCRIPT's output to standard out is redirected to
               # 'check_result' as required by normal check script.
               problem_home = ENV['PROBLEM_HOME']
               require "#{problem_home}/script/test_dsl.rb"
               if ARGV.length < 2
                 puts "Usage: check <language> <test-number> [<output-file>]"
                 exit(0)
               end
               language = ARGV[0]
               test_num = ARGV[1].to_i
               if ARGV.length >= 3
                 output_file_name = ARGV[2]
               else
                 output_file_name = "output.txt"
               end
               load "#{problem_home}/test_cases/all_tests.cfg"
               problem = Problem.get_instance
               answer_file_name = "#{problem_home}/test_cases/#{test_num}/answer-#{test_num}.txt"
               input_file_name = "#{problem_home}/test_cases/#{test_num}/input-#{test_num}.txt"
               score = problem.get_score(test_num)
               cmd = "#{problem_home}/script/#{REAL_CHECK_SCRIPT} " +
                 "#{language} #{test_num} #{input_file_name} #{output_file_name} " +
                 "#{answer_file_name} #{score} > check_result"
               #puts "wrapper-CMD: #{cmd}"
               system(cmd)

test/data/ev/test_memory/script/check ¶ +1 -1

-            - #!/usr/bin/ruby
+            + #!/usr/bin/env ruby
               problem_home = ENV['PROBLEM_HOME']
               require "#{problem_home}/script/test_dsl.rb"
               if ARGV.length < 2
                 puts "Usage: check <language> <test-number> [<output-file>]"
                 exit(0)
               end
               language = ARGV[0]
               test_num = ARGV[1].to_i
               if ARGV.length >= 3
                 output_file_name = ARGV[2]
               else
                 output_file_name = "output.txt"
               end
               load "#{problem_home}/test_cases/all_tests.cfg"
               problem = Problem.get_instance
               output_file = File.new(output_file_name, "r")
               answer_file = File.new("#{problem_home}/test_cases/#{test_num}/answer-#{test_num}.txt")
               result_file = File.new("check_result", "w")
               output_file_content = output_file.read
               answer_file_content = answer_file.read
               report_correct = lambda {
                 result_file.write "Correct\n"
                 result_file.write problem.get_score(test_num)
                 result_file.write "\n"
                 result_file.close
                 exit(0)
               }
               report_wrong = lambda {
                 result_file.write "Incorrect\n"
                 result_file.write "0\n"
                 result_file.close
                 exit(0)
               }
               ##################
               # Your code here #
               ##################
               num_pattern = /^[0-9]*/
               if (output_file_content =~ num_pattern) == nil
                 report_wrong.call
               end
               output_i = output_file_content.to_i
               answer_i = answer_file_content.to_i
               if output_i == answer_i
                 report_correct.call
               else
                 report_wrong.call
               end

test/data/ev/test_normal/script/check ¶ +1 -1

-            - #!/usr/bin/ruby
+            + #!/usr/bin/env ruby
               problem_home = ENV['PROBLEM_HOME']
               require "#{problem_home}/script/test_dsl.rb"
               if ARGV.length < 2
                 puts "Usage: check <language> <test-number> [<output-file>]"
                 exit(0)
               end
               language = ARGV[0]
               test_num = ARGV[1].to_i
               if ARGV.length >= 3
                 output_file_name = ARGV[2]
               else
                 output_file_name = "output.txt"
               end
               load "#{problem_home}/test_cases/all_tests.cfg"
               problem = Problem.get_instance
               output_file = File.new(output_file_name, "r")
               answer_file = File.new("#{problem_home}/test_cases/#{test_num}/answer-#{test_num}.txt")
               result_file = File.new("check_result", "w")
               output_file_content = output_file.read
               answer_file_content = answer_file.read
               report_correct = lambda {
                 result_file.write "Correct\n"
                 result_file.write problem.get_score(test_num)
                 result_file.write "\n"
                 result_file.close
                 exit(0)
               }
               report_wrong = lambda {
                 result_file.write "Incorrect\n"
                 result_file.write "0\n"
                 result_file.close
                 exit(0)
               }
               ##################
               # Your code here #
               ##################
               num_pattern = /^[0-9]*/
               if (output_file_content =~ num_pattern) == nil
                 report_wrong.call
               end
               output_i = output_file_content.to_i
               answer_i = answer_file_content.to_i
               if output_i == answer_i
                 report_correct.call
               else
                 report_wrong.call
               end

test/data/ev/test_timeout/script/check ¶ +1 -1

-            - #!/usr/bin/ruby
+            + #!/usr/bin/env ruby
               problem_home = ENV['PROBLEM_HOME']
               require "#{problem_home}/script/test_dsl.rb"
               if ARGV.length < 2
                 puts "Usage: check <language> <test-number> [<output-file>]"
                 exit(0)
               end
               language = ARGV[0]
               test_num = ARGV[1].to_i
               if ARGV.length >= 3
                 output_file_name = ARGV[2]
               else
                 output_file_name = "output.txt"
               end
               load "#{problem_home}/test_cases/all_tests.cfg"
               problem = Problem.get_instance
               output_file = File.new(output_file_name, "r")
               answer_file = File.new("#{problem_home}/test_cases/#{test_num}/answer-#{test_num}.txt")
               result_file = File.new("check_result", "w")
               output_file_content = output_file.read
               answer_file_content = answer_file.read
               report_correct = lambda {
                 result_file.write "Correct\n"
                 result_file.write problem.get_score(test_num)
                 result_file.write "\n"
                 result_file.close
                 exit(0)
               }
               report_wrong = lambda {
                 result_file.write "Incorrect\n"
                 result_file.write "0\n"
                 result_file.close
                 exit(0)
               }
               ##################
               # Your code here #
               ##################
               num_pattern = /^[0-9]*/
               if (output_file_content =~ num_pattern) == nil
                 report_wrong.call
               end
               output_i = output_file_content.to_i
               answer_i = answer_file_content.to_i
               if output_i == answer_i
                 report_correct.call
               else
                 report_wrong.call
               end

test/data/ev/test_yesno/script/check ¶ +1 -1

-            - #!/usr/bin/ruby
+            + #!/usr/bin/env ruby
               problem_home = ENV['PROBLEM_HOME']
               require "#{problem_home}/script/test_dsl.rb"
               if ARGV.length < 2
                 puts "Usage: check <language> <test-number> [<output-file>]"
                 exit(0)
               end
               language = ARGV[0]
               test_num = ARGV[1].to_i
               if ARGV.length >= 3
                 output_file_name = ARGV[2]
               else
                 output_file_name = "output.txt"
               end
               load "#{problem_home}/test_cases/all_tests.cfg"
               problem = Problem.get_instance
               output_file = File.new(output_file_name, "r")
               answer_file = File.new("#{problem_home}/test_cases/#{test_num}/answer-#{test_num}.txt")
               result_file = File.new("check_result", "w")
               output_file_content = output_file.read
               answer_file_content = answer_file.read
               report_correct = lambda {
                 result_file.write "Correct\n"
                 result_file.write problem.get_score(test_num)
                 result_file.write "\n"
                 result_file.close
                 exit(0)
               }
               report_wrong = lambda {
                 result_file.write "Incorrect\n"
                 result_file.write "0\n"
                 result_file.close
                 exit(0)
               }
               ##################
               # Your code here #
               ##################
               ########### THIS IS FOR CHECKING TEXT ##########
               # check visible text
               out_items = output_file_content.split
               ans_items = answer_file_content.split
               if out_items.length != ans_items.length
                 report_wrong.call
               else
                 out_items.length.times do |i|
                   if out_items[i]!=ans_items[i]
                     report_wrong.call
                   end
                 end
                 report_correct.call
               end

test/data/test_request/problems/test_normal/script/check ¶ +1 -1

-            - #!/usr/bin/ruby
+            + #!/usr/bin/env ruby
               problem_home = ENV['PROBLEM_HOME']
               require "#{problem_home}/script/test_dsl.rb"
               if ARGV.length < 2
                 puts "Usage: check <language> <test-number> [<output-file>]"
                 exit(0)
               end
               language = ARGV[0]
               test_num = ARGV[1].to_i
               if ARGV.length >= 3
                 output_file_name = ARGV[2]
               else
                 output_file_name = "output.txt"
               end
               load "#{problem_home}/test_cases/all_tests.cfg"
               problem = Problem.get_instance
               output_file = File.new(output_file_name, "r")
               answer_file = File.new("#{problem_home}/test_cases/#{test_num}/answer-#{test_num}.txt")
               result_file = File.new("check_result", "w")
               output_file_content = output_file.read
               answer_file_content = answer_file.read
               report_correct = lambda {
                 result_file.write "Correct\n"
                 result_file.write problem.get_score(test_num)
                 result_file.write "\n"
                 result_file.close
                 exit(0)
               }
               report_wrong = lambda {
                 result_file.write "Incorrect\n"
                 result_file.write "0\n"
                 result_file.close
                 exit(0)
               }
               ##################
               # Your code here #
               ##################
               report_correct.call

test/engine_spec.rb ¶ +4 -4

                 end
                 it "should not allow malicious submission to see PROBLEM_HOME" do
                   problem_test_yesno =  stub(Problem,
                                               :id => 1, :name => 'test_yesno',
                                               :full_score => 10)
                   grader_should(:grade => "yesno_access_problem_home.c",
                                 :on => problem_test_yesno,
                                 :and_report => {
                                   :score => 0,
                                   :comment => /(-|x)/})
                 end
                 it "should not allow malicious submission to open files" do
                   problem_test_yesno =  stub(Problem,
                                               :id => 1, :name => 'test_yesno',
                                               :full_score => 10)
                   grader_should(:grade => "yesno_open_file.c",
                                 :on => problem_test_yesno,
                                 :and_report => {
                                   :score => 0,
                                   :comment => /(-|x)/})
                 end
                 def grader_should(args)
                   @user1 = stub(User,
                                 :id => 1, :login => 'user1')
                   submission =
                     create_submission_from_file(1, @user1, args[:on], args[:grade], args[:language])
                   submission.should_receive(:graded_at=)
                   expected_score = args[:and_report][:score]
                   expected_comment = args[:and_report][:comment]
                   if args[:and_report][:compiler_message]!=nil
                     expected_compiler_message = args[:and_report][:compiler_message]
                   else
                     expected_compiler_message = ''
                   end
                   submission.should_receive(:points=).with(expected_score)
                   submission.should_receive(:grader_comment=).with(expected_comment)
                   submission.should_receive(:compiler_message=).with(expected_compiler_message)
                   submission.should_receive(:save)
                   @engine.grade(submission)
                 end
                 protected
                 def create_normal_submission_mock_from_file(source_fname)
                   create_submission_from_file(1, @user_user1, @problem_test_normal, source_fname)
                 end
               end
               describe "A grader engine, when grading test requests" do
                 include GraderEngineHelperMethods
                 before(:each) do
                   @config = Grader::Configuration.get_instance
                   @engine = Grader::Engine.new(:room_maker => Grader::TestRequestRoomMaker.new,
                                                :reporter => Grader::TestRequestReporter.new)
                   init_sandbox
                 end
                 it "should report error if there is no problem template" do
                   problem = stub(Problem,
                                  :id => 1, :name => 'nothing')
                   grader_should(:grade => 'test1_correct.c',
                                 :on => problem,
                                 :with => 'in1.txt',
                                 :and_report => {
                                   :graded_at= => nil,
                                   :compiler_message= => '',
                                   :grader_comment= => '',
                                   :running_stat= => /template not found/,
                                   :running_time= => nil,
                                   :exit_status= => nil,
                                   :memory_usage= => nil,
                                   :save => nil})
                 end
                 it "should run test request and produce output file" do
                   problem = stub(Problem,
                                  :id => 1, :name => 'test_normal')
                   grader_should(:grade => 'test1_correct.c',
                                 :on => problem,
                                 :with => 'in1.txt',
                                 :and_report => {
                                   :graded_at= => nil,
                                   :compiler_message= => '',
                                   :grader_comment= => '',
                                   :running_stat= => /0.0\d* sec./,
                                   :output_file_name= => lambda { |fname|
-            -                       File.exists?(fname).should be_true
+            +                       File.exists?(fname).should be true
                                   },
                                   :running_time= => nil,
                                   :exit_status= => nil,
                                   :memory_usage= => nil,
                                   :save => nil})
                 end
                 it "should clean up problem directory after running test request" do
                   problem = stub(Problem,
                                  :id => 1, :name => 'test_normal')
                   grader_should(:grade => 'test1_correct.c',
                                 :on => problem,
                                 :with => 'in1.txt',
                                 :and_report => {
                                   :graded_at= => nil,
                                   :compiler_message= => '',
                                   :grader_comment= => '',
                                   :running_stat= => nil,
                                   :output_file_name= => nil,
                                   :running_time= => nil,
                                   :exit_status= => nil,
                                   :memory_usage= => nil,
                                   :save => nil})
-            -     File.exists?(@config.user_result_dir + "/test_request/test_normal/test_cases/1/input-1.txt").should be_false
+            +     File.exists?(@config.user_result_dir + "/test_request/test_normal/test_cases/1/input-1.txt").should be false
                 end
                 it "should compile test request with error and report compilation error" do
                   problem = stub(Problem,
                                  :id => 1, :name => 'test_normal')
                   grader_should(:grade => 'test1_compile_error.c',
                                 :on => problem,
                                 :with => 'in1.txt',
                                 :and_report => {
                                   :graded_at= => nil,
                                   :compiler_message= => /.+/,
                                   :grader_comment= => /[Cc]ompil.*error/,
                                   :running_stat= => '',
                                   :save => nil})
                 end
                 it "should report exit status" do
                   problem = stub(Problem,
                                  :id => 1, :name => 'test_normal')
                   grader_should(:grade => 'add_nonzero_exit_status.c',
                                 :on => problem,
                                 :with => 'in1.txt',
                                 :and_report => {
                                   :graded_at= => nil,
                                   :compiler_message= => '',
                                   :grader_comment= => '',
                                   :running_stat= => /[Ee]xit.*status.*10.*0\.0\d* sec/m,
                                   :output_file_name= => lambda { |fname|
-            -                       File.exists?(fname).should be_true
+            +                       File.exists?(fname).should be true
                                   },
                                   :running_time= => nil,
                                   :exit_status= => /10/,
                                   :memory_usage= => nil,
                                   :save => nil})
                 end
                 it "should produce running statistics for normal submission" do
                   problem = stub(Problem,
                                  :id => 1, :name => 'test_normal')
                   grader_should(:grade => 'test_run_stat.c',
                                 :on => problem,
                                 :with => 'in1.txt',
                                 :and_report => {
                                   :graded_at= => nil,
                                   :compiler_message= => '',
                                   :grader_comment= => '',
                                   :running_stat= => nil,
                                   :output_file_name= => lambda { |fname|
-            -                       File.exists?(fname).should be_true
+            +                       File.exists?(fname).should be true
                                   },
                                   :running_time= => lambda { |t|
                                     (t>=0.14) and (t<=0.16)
                                   },
                                   :exit_status= => nil,
                                   :memory_usage= => lambda { |s|
                                     (s>=500) and (s<=1000)
                                   },
                                   :save => nil})
                 end
                 protected
                 def grader_should(args)
                   @user1 = stub(User,
                                 :id => 1, :login => 'user1')
                   problem = args[:on]
                   input_file = @config.test_request_input_base_dir + "/" + args[:with]
                   submission =
                     create_submission_from_file(1, @user1, args[:on], args[:grade])
                   test_request = stub(TestRequest,
                                       :id => 1,
                                       :user => @user1,
                                       :problem => problem,
                                       :submission => submission,
                                       :input_file_name => input_file,
                                       :language => submission.language,
                                       :problem_name => problem.name)
                   expectations = args[:and_report]
                   expectations.each do |key,val|
                     if val==nil
                       test_request.should_receive(key)
                     elsif val.class == Proc
                       test_request.should_receive(key) { |fname|
                         val.call(fname)
                       }
                     else
                       test_request.should_receive(key).with(val)
                     end
                   end
                   @engine.grade(test_request)
                 end
               end

Write
Preview

You need to be logged in to leave comments. Login now

	Site-wide shortcuts
/	Use quick search box
g h	Goto home page
g g	Goto my private gists page
g G	Goto my public gists page
n r	New repository page
n g	New gist page

	Repositories
g s	Goto summary page
g c	Goto changelog page
g f	Goto files page
g F	Goto files page with file search activated
g p	Goto pull requests page
g o	Goto repository settings
g O	Goto repository permissions settings

Sign in to your account

Author

r206:cf0534046f00 - Thu, 01 Jan 2015 02:08:47 - 8 files changed: 11 inserted, 11 deleted

Sign in to your account

Commit r206:cf0534046f00 public

Author

r206:cf0534046f00 - Thu, 01 Jan 2015 02:08:47 - 8 files changed: 11 inserted, 11 deleted

Commit `r206:cf0534046f00` public