cafe-grader-judge-scripts

Location: cafe-grader-judge-scripts/test/engine_spec.rb - annotation

Show More
Commit Description:
[grader] import script now support raw with testruns...
Commit Description:
[grader] import script now support raw with testruns git-svn-id: http://theory.cpe.ku.ac.th/grader/judge/trunk/scripts@267 6386c4cd-e34a-4fa8-8920-d93eb39b512e
References:
r63:031546561018 default
File last commit:
r51:3618759afb50 default
Show/Diff file:
Action:
test/engine_spec.rb | 316 lines | 11.2 KiB | text/x-ruby | RubyLexer |
History | Source | Raw | Download
jittat
[grader] [MERGED] Merged new-arch-branch changes 74:105 into the trunk...
 r23 require File.join(File.dirname(__FILE__),'spec_helper')
require File.join(File.dirname(__FILE__),'engine_spec_helper')
describe "A grader engine, when grading submissions" do
include GraderEngineHelperMethods
before(:each) do
@config = Grader::Configuration.get_instance
# this test is from Pong
@problem_test_normal = stub(Problem,
:id => 1, :name => 'test_normal',
:full_score => 135)
@user_user1 = stub(User,
:id => 1, :login => 'user1')
@engine = Grader::Engine.new
init_sandbox
end
it "should grade normal submission" do
grader_should(:grade => "test1_correct.c",
:on => @problem_test_normal,
:and_report => {
:score => 135,
:comment => /^PASSED/})
end
it "should produce error message when submission cannot compile" do
grader_should(:grade => "test1_compile_error.c",
:on => @problem_test_normal,
:and_report => {
:score => 0,
jittat
[grader] fixed bug #6 and #9...
 r37 :comment => 'FAILED: compilation error',
jittat
[grader] [MERGED] Merged new-arch-branch changes 74:105 into the trunk...
 r23 :compiler_message => /[Ee]rror/})
end
it "should produce timeout error when submission runs forever" do
@problem_test_timeout = stub(Problem,
:id => 1, :name => 'test_timeout',
:full_score => 10)
grader_should(:grade => "test2_timeout.c",
:on => @problem_test_timeout,
:and_report => {
:score => 0,
:comment => 'FAILED: TT'})
end
jittat
[grader] added fractional timelimit...
 r51 it "should produce timeout error correctly with fractional running time and fractional time limits" do
@problem_test_timeout = stub(Problem,
jittat
[grader] [MERGED] Merged new-arch-branch changes 74:105 into the trunk...
 r23 :id => 1, :name => 'test_timeout',
:full_score => 20)
jittat
[grader] added fractional timelimit...
 r51 grader_should(:grade => "test2_05sec.c",
jittat
[grader] [MERGED] Merged new-arch-branch changes 74:105 into the trunk...
 r23 :on => @problem_test_timeout,
:and_report => {
:score => 10,
:comment => 'FAILED: TP'})
end
it "should produce runtime error when submission uses too much static memory" do
@problem_test_memory = stub(Problem,
:id => 1, :name => 'test_memory',
:full_score => 20)
grader_should(:grade => "add_too_much_memory_static.c",
:on => @problem_test_memory,
:and_report => {
:score => 10,
:comment => /FAILED: [^P]P/})
end
it "should not allow submission to allocate too much dynamic memory" do
@problem_test_memory = stub(Problem,
:id => 1, :name => 'test_memory',
:full_score => 20)
grader_should(:grade => "add_too_much_memory_dynamic.c",
:on => @problem_test_memory,
:and_report => {
:score => 10,
:comment => /FAILED: [^P]P/})
end
jittat
fix test run scoring bug...
 r32 it "should score test runs correctly when submission fails in some test case" do
grader_should(:grade => "add_fail_test_case_1.c",
:on => @problem_test_normal,
:and_report => {
:score => 105,
:comment => /^FAILED:/})
end
jittat
[grader] [MERGED] Merged new-arch-branch changes 74:105 into the trunk...
 r23 it "should fail submission with non-zero exit status" do
jittat
[grader] finish engine spec on grading non-zero exit status...
 r33 grader_should(:grade => "add_nonzero_exit_status.c",
:on => @problem_test_normal,
:and_report => {
:score => 0,
:comment => /^FAILED:/})
jittat
[grader] [MERGED] Merged new-arch-branch changes 74:105 into the trunk...
 r23 end
jittat
[grader] new test on accessing PROBLEM_HOME and openning files...
 r35 it "should not allow malicious submission to see PROBLEM_HOME" do
problem_test_yesno = stub(Problem,
:id => 1, :name => 'test_yesno',
:full_score => 10)
grader_should(:grade => "yesno_access_problem_home.c",
:on => problem_test_yesno,
:and_report => {
:score => 0,
:comment => /^FAILED:/})
end
it "should not allow malicious submission to open files" do
problem_test_yesno = stub(Problem,
:id => 1, :name => 'test_yesno',
:full_score => 10)
grader_should(:grade => "yesno_open_file.c",
:on => problem_test_yesno,
:and_report => {
:score => 0,
:comment => /^FAILED:/})
end
jittat
[grader] [MERGED] Merged new-arch-branch changes 74:105 into the trunk...
 r23 def grader_should(args)
@user1 = stub(User,
:id => 1, :login => 'user1')
submission =
create_submission_from_file(1, @user1, args[:on], args[:grade])
submission.should_receive(:graded_at=)
expected_score = args[:and_report][:score]
expected_comment = args[:and_report][:comment]
if args[:and_report][:compiler_message]!=nil
expected_compiler_message = args[:and_report][:compiler_message]
else
expected_compiler_message = ''
end
submission.should_receive(:points=).with(expected_score)
submission.should_receive(:grader_comment=).with(expected_comment)
submission.should_receive(:compiler_message=).with(expected_compiler_message)
submission.should_receive(:save)
@engine.grade(submission)
end
protected
def create_normal_submission_mock_from_file(source_fname)
create_submission_from_file(1, @user_user1, @problem_test_normal, source_fname)
end
end
describe "A grader engine, when grading test requests" do
include GraderEngineHelperMethods
before(:each) do
@config = Grader::Configuration.get_instance
@engine = Grader::Engine.new(Grader::TestRequestRoomMaker.new,
Grader::TestRequestReporter.new)
init_sandbox
end
it "should report error if there is no problem template" do
problem = stub(Problem,
:id => 1, :name => 'nothing')
grader_should(:grade => 'test1_correct.c',
:on => problem,
:with => 'in1.txt',
:and_report => {
:graded_at= => nil,
:compiler_message= => '',
:grader_comment= => '',
:running_stat= => /template not found/,
jittat
[grader] extracted test_request info into new stat columns...
 r41 :running_time= => nil,
:exit_status= => nil,
:memory_usage= => nil,
jittat
[grader] [MERGED] Merged new-arch-branch changes 74:105 into the trunk...
 r23 :save => nil})
end
it "should run test request and produce output file" do
problem = stub(Problem,
:id => 1, :name => 'test_normal')
grader_should(:grade => 'test1_correct.c',
:on => problem,
:with => 'in1.txt',
:and_report => {
:graded_at= => nil,
:compiler_message= => '',
:grader_comment= => '',
jittat
[grader] small tweak on one test case...
 r44 :running_stat= => /0.0\d* sec./,
jittat
[grader] [MERGED] Merged new-arch-branch changes 74:105 into the trunk...
 r23 :output_file_name= => lambda { |fname|
File.exists?(fname).should be_true
},
jittat
[grader] extracted test_request info into new stat columns...
 r41 :running_time= => nil,
:exit_status= => nil,
:memory_usage= => nil,
jittat
[grader] [MERGED] Merged new-arch-branch changes 74:105 into the trunk...
 r23 :save => nil})
end
it "should clean up problem directory after running test request" do
problem = stub(Problem,
:id => 1, :name => 'test_normal')
grader_should(:grade => 'test1_correct.c',
:on => problem,
:with => 'in1.txt',
:and_report => {
:graded_at= => nil,
:compiler_message= => '',
:grader_comment= => '',
:running_stat= => nil,
:output_file_name= => nil,
jittat
[grader] extracted test_request info into new stat columns...
 r41 :running_time= => nil,
:exit_status= => nil,
:memory_usage= => nil,
jittat
[grader] [MERGED] Merged new-arch-branch changes 74:105 into the trunk...
 r23 :save => nil})
File.exists?(@config.user_result_dir + "/test_request/test_normal/test_cases/1/input-1.txt").should be_false
end
it "should compile test request with error and report compilation error" do
problem = stub(Problem,
:id => 1, :name => 'test_normal')
grader_should(:grade => 'test1_compile_error.c',
:on => problem,
:with => 'in1.txt',
:and_report => {
:graded_at= => nil,
:compiler_message= => /.+/,
:grader_comment= => /[Cc]ompil.*error/,
:running_stat= => '',
:save => nil})
end
it "should report exit status" do
problem = stub(Problem,
:id => 1, :name => 'test_normal')
grader_should(:grade => 'add_nonzero_exit_status.c',
:on => problem,
:with => 'in1.txt',
:and_report => {
:graded_at= => nil,
:compiler_message= => '',
:grader_comment= => '',
jittat
[grader] box.cc now reports running status when die...
 r46 :running_stat= => /[Ee]xit.*status.*10.*0\.0\d* sec/m,
jittat
[grader] [MERGED] Merged new-arch-branch changes 74:105 into the trunk...
 r23 :output_file_name= => lambda { |fname|
File.exists?(fname).should be_true
},
jittat
[grader] extracted test_request info into new stat columns...
 r41 :running_time= => nil,
:exit_status= => /10/,
:memory_usage= => nil,
jittat
[grader] [MERGED] Merged new-arch-branch changes 74:105 into the trunk...
 r23 :save => nil})
end
jittat
[grader] added memory usage status report, better running time report...
 r42 it "should produce running statistics for normal submission" do
problem = stub(Problem,
:id => 1, :name => 'test_normal')
grader_should(:grade => 'test_run_stat.c',
:on => problem,
:with => 'in1.txt',
:and_report => {
:graded_at= => nil,
:compiler_message= => '',
:grader_comment= => '',
:running_stat= => nil,
:output_file_name= => lambda { |fname|
File.exists?(fname).should be_true
},
:running_time= => lambda { |t|
(t>=0.14) and (t<=0.16)
},
:exit_status= => nil,
:memory_usage= => lambda { |s|
(s>=500) and (s<=1000)
},
:save => nil})
end
jittat
[grader] [MERGED] Merged new-arch-branch changes 74:105 into the trunk...
 r23 protected
def grader_should(args)
@user1 = stub(User,
:id => 1, :login => 'user1')
problem = args[:on]
input_file = @config.test_request_input_base_dir + "/" + args[:with]
submission =
create_submission_from_file(1, @user1, args[:on], args[:grade])
test_request = stub(TestRequest,
:id => 1,
:user => @user1,
:problem => problem,
:submission => submission,
:input_file_name => input_file,
:language => submission.language,
:problem_name => problem.name)
expectations = args[:and_report]
expectations.each do |key,val|
if val==nil
test_request.should_receive(key)
elsif val.class == Proc
test_request.should_receive(key) { |fname|
val.call(fname)
}
else
test_request.should_receive(key).with(val)
end
end
@engine.grade(test_request)
end
end