diff --git a/judge/scripts/test/engine_spec.rb b/judge/scripts/test/engine_spec.rb new file mode 100644 --- /dev/null +++ b/judge/scripts/test/engine_spec.rb @@ -0,0 +1,316 @@ +require File.join(File.dirname(__FILE__),'spec_helper') +require File.join(File.dirname(__FILE__),'engine_spec_helper') + +describe "A grader engine, when grading submissions" do + + include GraderEngineHelperMethods + + before(:each) do + @config = Grader::Configuration.get_instance + + # this test is from Pong + @problem_test_normal = stub(Problem, + :id => 1, :name => 'test_normal', + :full_score => 135) + @user_user1 = stub(User, + :id => 1, :login => 'user1') + + @engine = Grader::Engine.new + init_sandbox + end + + it "should grade normal submission" do + grader_should(:grade => "test1_correct.c", + :on => @problem_test_normal, + :and_report => { + :score => 135, + :comment => /^(\[|P|\])+/}) + end + + + it "should produce error message when submission cannot compile" do + grader_should(:grade => "test1_compile_error.c", + :on => @problem_test_normal, + :and_report => { + :score => 0, + :comment => 'compilation error', + :compiler_message => /[Ee]rror/}) + end + + it "should produce timeout error when submission runs forever" do + @problem_test_timeout = stub(Problem, + :id => 1, :name => 'test_timeout', + :full_score => 10) + grader_should(:grade => "test2_timeout.c", + :on => @problem_test_timeout, + :and_report => { + :score => 0, + :comment => 'TT'}) + end + + it "should produce timeout error correctly with fractional running time and fractional time limits" do + @problem_test_timeout = stub(Problem, + :id => 1, :name => 'test_timeout', + :full_score => 20) + grader_should(:grade => "test2_05sec.c", + :on => @problem_test_timeout, + :and_report => { + :score => 10, + :comment => 'TP'}) + end + + it "should produce runtime error when submission uses too much static memory" do + @problem_test_memory = stub(Problem, + :id => 1, :name => 'test_memory', + :full_score => 20) + grader_should(:grade => "add_too_much_memory_static.c", + :on => @problem_test_memory, + :and_report => { + :score => 10, + :comment => /[^P]P/}) + end + + it "should not allow submission to allocate too much dynamic memory" do + @problem_test_memory = stub(Problem, + :id => 1, :name => 'test_memory', + :full_score => 20) + grader_should(:grade => "add_too_much_memory_dynamic.c", + :on => @problem_test_memory, + :and_report => { + :score => 10, + :comment => /[^P]P/}) + end + + it "should score test runs correctly when submission fails in some test case" do + grader_should(:grade => "add_fail_test_case_1.c", + :on => @problem_test_normal, + :and_report => { + :score => 105, + :comment => /^.*(-|x).*$/}) + end + + it "should fail submission with non-zero exit status" do + grader_should(:grade => "add_nonzero_exit_status.c", + :on => @problem_test_normal, + :and_report => { + :score => 0, + :comment => /^(\[|x|\])+$/}) + end + + it "should not allow malicious submission to see PROBLEM_HOME" do + problem_test_yesno = stub(Problem, + :id => 1, :name => 'test_yesno', + :full_score => 10) + grader_should(:grade => "yesno_access_problem_home.c", + :on => problem_test_yesno, + :and_report => { + :score => 0, + :comment => /(-|x)/}) + end + + it "should not allow malicious submission to open files" do + problem_test_yesno = stub(Problem, + :id => 1, :name => 'test_yesno', + :full_score => 10) + grader_should(:grade => "yesno_open_file.c", + :on => problem_test_yesno, + :and_report => { + :score => 0, + :comment => /(-|x)/}) + end + + def grader_should(args) + @user1 = stub(User, + :id => 1, :login => 'user1') + submission = + create_submission_from_file(1, @user1, args[:on], args[:grade]) + submission.should_receive(:graded_at=) + + expected_score = args[:and_report][:score] + expected_comment = args[:and_report][:comment] + if args[:and_report][:compiler_message]!=nil + expected_compiler_message = args[:and_report][:compiler_message] + else + expected_compiler_message = '' + end + + submission.should_receive(:points=).with(expected_score) + submission.should_receive(:grader_comment=).with(expected_comment) + submission.should_receive(:compiler_message=).with(expected_compiler_message) + submission.should_receive(:save) + + @engine.grade(submission) + end + + protected + + def create_normal_submission_mock_from_file(source_fname) + create_submission_from_file(1, @user_user1, @problem_test_normal, source_fname) + end + +end + +describe "A grader engine, when grading test requests" do + + include GraderEngineHelperMethods + + before(:each) do + @config = Grader::Configuration.get_instance + @engine = Grader::Engine.new(Grader::TestRequestRoomMaker.new, + Grader::TestRequestReporter.new) + init_sandbox + end + + it "should report error if there is no problem template" do + problem = stub(Problem, + :id => 1, :name => 'nothing') + grader_should(:grade => 'test1_correct.c', + :on => problem, + :with => 'in1.txt', + :and_report => { + :graded_at= => nil, + :compiler_message= => '', + :grader_comment= => '', + :running_stat= => /template not found/, + :running_time= => nil, + :exit_status= => nil, + :memory_usage= => nil, + :save => nil}) + end + + it "should run test request and produce output file" do + problem = stub(Problem, + :id => 1, :name => 'test_normal') + grader_should(:grade => 'test1_correct.c', + :on => problem, + :with => 'in1.txt', + :and_report => { + :graded_at= => nil, + :compiler_message= => '', + :grader_comment= => '', + :running_stat= => /0.0\d* sec./, + :output_file_name= => lambda { |fname| + File.exists?(fname).should be_true + }, + :running_time= => nil, + :exit_status= => nil, + :memory_usage= => nil, + :save => nil}) + end + + it "should clean up problem directory after running test request" do + problem = stub(Problem, + :id => 1, :name => 'test_normal') + grader_should(:grade => 'test1_correct.c', + :on => problem, + :with => 'in1.txt', + :and_report => { + :graded_at= => nil, + :compiler_message= => '', + :grader_comment= => '', + :running_stat= => nil, + :output_file_name= => nil, + :running_time= => nil, + :exit_status= => nil, + :memory_usage= => nil, + :save => nil}) + File.exists?(@config.user_result_dir + "/test_request/test_normal/test_cases/1/input-1.txt").should be_false + end + + it "should compile test request with error and report compilation error" do + problem = stub(Problem, + :id => 1, :name => 'test_normal') + grader_should(:grade => 'test1_compile_error.c', + :on => problem, + :with => 'in1.txt', + :and_report => { + :graded_at= => nil, + :compiler_message= => /.+/, + :grader_comment= => /[Cc]ompil.*error/, + :running_stat= => '', + :save => nil}) + end + + it "should report exit status" do + problem = stub(Problem, + :id => 1, :name => 'test_normal') + grader_should(:grade => 'add_nonzero_exit_status.c', + :on => problem, + :with => 'in1.txt', + :and_report => { + :graded_at= => nil, + :compiler_message= => '', + :grader_comment= => '', + :running_stat= => /[Ee]xit.*status.*10.*0\.0\d* sec/m, + :output_file_name= => lambda { |fname| + File.exists?(fname).should be_true + }, + :running_time= => nil, + :exit_status= => /10/, + :memory_usage= => nil, + :save => nil}) + end + + it "should produce running statistics for normal submission" do + problem = stub(Problem, + :id => 1, :name => 'test_normal') + grader_should(:grade => 'test_run_stat.c', + :on => problem, + :with => 'in1.txt', + :and_report => { + :graded_at= => nil, + :compiler_message= => '', + :grader_comment= => '', + :running_stat= => nil, + :output_file_name= => lambda { |fname| + File.exists?(fname).should be_true + }, + :running_time= => lambda { |t| + (t>=0.14) and (t<=0.16) + }, + :exit_status= => nil, + :memory_usage= => lambda { |s| + (s>=500) and (s<=1000) + }, + :save => nil}) + end + + protected + def grader_should(args) + @user1 = stub(User, + :id => 1, :login => 'user1') + + problem = args[:on] + input_file = @config.test_request_input_base_dir + "/" + args[:with] + + submission = + create_submission_from_file(1, @user1, args[:on], args[:grade]) + + test_request = stub(TestRequest, + :id => 1, + :user => @user1, + :problem => problem, + :submission => submission, + :input_file_name => input_file, + :language => submission.language, + :problem_name => problem.name) + + expectations = args[:and_report] + + expectations.each do |key,val| + if val==nil + test_request.should_receive(key) + elsif val.class == Proc + test_request.should_receive(key) { |fname| + val.call(fname) + } + else + test_request.should_receive(key).with(val) + end + end + + @engine.grade(test_request) + end + +end +