cafe-grader-judge-scripts Commit

cafe-grader-judge-scripts

Commit `r39:7a9b64601b9d` public

parent | child

Description:

[grader] engine_spec: fix reg-ex new line problem, change test case for 1.5 sec submission (as the machine is a little faster) git-svn-id: http://theory.cpe.ku.ac.th/grader/judge/trunk/scripts@150 6386c4cd-e34a-4fa8-8920-d93eb39b512e

Commit status:

[Not Reviewed]

References:

default

Diff options:

| | | |

Comments:

0 Commit comments 0 Inline Comments

Unresolved TODOs:

There are no unresolved TODOs

Add another comment

r39:7a9b64601b9d - Sun, 23 Mar 2008 09:12:41 - 3 files changed: 3 inserted, 3 deleted

test/data/test2_1-5sec.c ¶ +1 -1

               #include <stdio.h>
               #include <stdlib.h>
               int main()
               {
                 int a,b;
                 int c=0;
                 scanf("%d %d",&a,&b);
                 printf("%d\n",a+b);
                 for(a=0; a<2; a++) {
-            -     while(c<1150000000) {
+            +     while(c<1550000000) {
                     c++;
                     b+=c;
                   }
                 }
                 exit(0);
               }

test/engine_spec.rb ¶ +1 -1

                                                :full_score => 10)
                   grader_should(:grade => "test2_timeout.c",
                                 :on => @problem_test_timeout,
                                 :and_report => {
                                   :score => 0,
                                   :comment => 'FAILED: TT'})
                 end
                 it "should produce timeout error correctly when submission runs slower than expected in less than a second" do
                   @problem_test_timeout = stub(Problem,
                                                :id => 1, :name => 'test_timeout',
                                                :full_score => 20)
                   grader_should(:grade => "test2_1-5sec.c",
                                 :on => @problem_test_timeout,
                                 :and_report => {
                                   :score => 10,
                                   :comment => 'FAILED: TP'})
                 end
                 it "should produce runtime error when submission uses too much static memory" do
                   @problem_test_memory = stub(Problem,
                                               :id => 1, :name => 'test_memory',
                                               :full_score => 20)
                   grader_should(:grade => "add_too_much_memory_static.c",
                                 :on => @problem_test_memory,
                                 :and_report => {
                                   :score => 10,
                                   :comment => /FAILED: [^P]P/})
                 end
                 it "should not allow submission to allocate too much dynamic memory" do
                   @problem_test_memory = stub(Problem,
                                               :id => 1, :name => 'test_memory',
                                               :full_score => 20)
                   grader_should(:grade => "add_too_much_memory_dynamic.c",
                                 :on => @problem_test_memory,
                                 :and_report => {
                                   :score => 10,
                                   :comment => /FAILED: [^P]P/})
                 end
                 it "should score test runs correctly when submission fails in some test case" do
                   grader_should(:grade => "add_fail_test_case_1.c",
                                 :on => @problem_test_normal,
                                 :and_report => {
                                   :score => 105,
                                   :comment => /^FAILED:/})
                 end
                 it "should fail submission with non-zero exit status" do
                   grader_should(:grade => "add_nonzero_exit_status.c",
                                 :on => @problem_test_normal,
                                 :and_report => {
                                   :score => 0,
                                   :comment => /^FAILED:/})
                 end
                 it "should not allow malicious submission to see PROBLEM_HOME" do
                   problem_test_yesno =  stub(Problem,
                                               :id => 1, :name => 'test_yesno',
                                               :full_score => 10)
                   grader_should(:grade => "yesno_access_problem_home.c",
                                 :on => problem_test_yesno,
                                 :and_report => {
                                   :score => 0,
                                   :comment => /^FAILED:/})
                 end
                 it "should not allow malicious submission to open files" do
                   problem_test_yesno =  stub(Problem,
                                               :id => 1, :name => 'test_yesno',
                                               :full_score => 10)
                   grader_should(:grade => "yesno_open_file.c",
                                 :on => problem_test_yesno,
                                 :and_report => {
                                   :score => 0,
                                   :comment => /^FAILED:/})
                 end
                 def grader_should(args)
                   @user1 = stub(User,
                                 :id => 1, :login => 'user1')
                   submission =
                     create_submission_from_file(1, @user1, args[:on], args[:grade])
                   submission.should_receive(:graded_at=)
                   expected_score = args[:and_report][:score]
                   expected_comment = args[:and_report][:comment]
                   if args[:and_report][:compiler_message]!=nil
                     expected_compiler_message = args[:and_report][:compiler_message]
                   else
                     expected_compiler_message = ''
                   end
                   submission.should_receive(:points=).with(expected_score)
                   submission.should_receive(:grader_comment=).with(expected_comment)
                   submission.should_receive(:compiler_message=).with(expected_compiler_message)
                   submission.should_receive(:save)
                   @engine.grade(submission)
                 end
                 protected
                 def create_normal_submission_mock_from_file(source_fname)
                   create_submission_from_file(1, @user_user1, @problem_test_normal, source_fname)
                 end
               end
               describe "A grader engine, when grading test requests" do
                 include GraderEngineHelperMethods
                 before(:each) do
                   @config = Grader::Configuration.get_instance
                   @engine = Grader::Engine.new(Grader::TestRequestRoomMaker.new,
                                                Grader::TestRequestReporter.new)
                   init_sandbox
                 end
                 it "should report error if there is no problem template" do
                   problem = stub(Problem,
                                  :id => 1, :name => 'nothing')
                   grader_should(:grade => 'test1_correct.c',
                                 :on => problem,
                                 :with => 'in1.txt',
                                 :and_report => {
                                   :graded_at= => nil,
                                   :compiler_message= => '',
                                   :grader_comment= => '',
                                   :running_stat= => /template not found/,
                                   :save => nil})
                 end
                 it "should run test request and produce output file" do
                   problem = stub(Problem,
                                  :id => 1, :name => 'test_normal')
                   grader_should(:grade => 'test1_correct.c',
                                 :on => problem,
                                 :with => 'in1.txt',
                                 :and_report => {
                                   :graded_at= => nil,
                                   :compiler_message= => '',
                                   :grader_comment= => '',
                                   :running_stat= => /0.0 sec./,
                                   :output_file_name= => lambda { |fname|
                                     File.exists?(fname).should be_true
                                   },
                                   :save => nil})
                 end
                 it "should clean up problem directory after running test request" do
                   problem = stub(Problem,
                                  :id => 1, :name => 'test_normal')
                   grader_should(:grade => 'test1_correct.c',
                                 :on => problem,
                                 :with => 'in1.txt',
                                 :and_report => {
                                   :graded_at= => nil,
                                   :compiler_message= => '',
                                   :grader_comment= => '',
                                   :running_stat= => nil,
                                   :output_file_name= => nil,
                                   :save => nil})
                   File.exists?(@config.user_result_dir + "/test_request/test_normal/test_cases/1/input-1.txt").should be_false
                 end
                 it "should compile test request with error and report compilation error" do
                   problem = stub(Problem,
                                  :id => 1, :name => 'test_normal')
                   grader_should(:grade => 'test1_compile_error.c',
                                 :on => problem,
                                 :with => 'in1.txt',
                                 :and_report => {
                                   :graded_at= => nil,
                                   :compiler_message= => /.+/,
                                   :grader_comment= => /[Cc]ompil.*error/,
                                   :running_stat= => '',
                                   :save => nil})
                 end
                 it "should report exit status" do
                   problem = stub(Problem,
                                  :id => 1, :name => 'test_normal')
                   grader_should(:grade => 'add_nonzero_exit_status.c',
                                 :on => problem,
                                 :with => 'in1.txt',
                                 :and_report => {
                                   :graded_at= => nil,
                                   :compiler_message= => '',
                                   :grader_comment= => '',
-            -                     :running_stat= => /[Ee]xit.*status.*10.*0.0 sec./,
+            +                     :running_stat= => /[Ee]xit.*status.*10.*0\.0 sec/m,
                                   :output_file_name= => lambda { |fname|
                                     File.exists?(fname).should be_true
                                   },
                                   :save => nil})
                 end
                 protected
                 def grader_should(args)
                   @user1 = stub(User,
                                 :id => 1, :login => 'user1')
                   problem = args[:on]
                   input_file = @config.test_request_input_base_dir + "/" + args[:with]
                   submission =
                     create_submission_from_file(1, @user1, args[:on], args[:grade])
                   test_request = stub(TestRequest,
                                       :id => 1,
                                       :user => @user1,
                                       :problem => problem,
                                       :submission => submission,
                                       :input_file_name => input_file,
                                       :language => submission.language,
                                       :problem_name => problem.name)
                   expectations = args[:and_report]
                   expectations.each do |key,val|
                     if val==nil
                       test_request.should_receive(key)
                     elsif val.class == Proc
                       test_request.should_receive(key) { |fname|
                         val.call(fname)
                       }
                     else
                       test_request.should_receive(key).with(val)
                     end
                   end
                   @engine.grade(test_request)
                 end
               end

test/engine_spec_helper.rb ¶ +1 -1

               module GraderEngineHelperMethods
                 def clear_sandbox
                   config = Grader::Configuration.get_instance
                   clear_cmd = "rm -rf #{config.test_sandbox_dir}/*"
                   system(clear_cmd)
                 end
                 def init_sandbox
                   config = Grader::Configuration.get_instance
                   clear_sandbox
-            -     Dir.mkdir config.user_result_dir
+            +     FileUtils.mkdir_p config.user_result_dir
                   cp_cmd = "cp -R #{config.test_data_dir}/ev #{config.test_sandbox_dir}"
                   system(cp_cmd)
                 end
                 def create_submission_from_file(id, user, problem,
                                                 source_fname, language=nil)
                   language = stub(Language, :name => 'c', :ext => 'c') if language==nil
                   config = Grader::Configuration.get_instance
                   source = File.open(config.test_data_dir + "/" + source_fname).read
                   stub(Submission,
                        :id => id, :user => user, :problem => problem,
                        :source => source, :language => language)
                 end
               end

Write
Preview

You need to be logged in to leave comments. Login now

	Site-wide shortcuts
/	Use quick search box
g h	Goto home page
g g	Goto my private gists page
g G	Goto my public gists page
n r	New repository page
n g	New gist page

	Repositories
g s	Goto summary page
g c	Goto changelog page
g f	Goto files page
g F	Goto files page with file search activated
g p	Goto pull requests page
g o	Goto repository settings
g O	Goto repository permissions settings

Sign in to your account

Author

r39:7a9b64601b9d - Sun, 23 Mar 2008 09:12:41 - 3 files changed: 3 inserted, 3 deleted

Sign in to your account

Commit r39:7a9b64601b9d public

Author

r39:7a9b64601b9d - Sun, 23 Mar 2008 09:12:41 - 3 files changed: 3 inserted, 3 deleted

Commit `r39:7a9b64601b9d` public