cafe-grader-judge-scripts

Location: cafe-grader-judge-scripts/test/engine_spec.rb - annotation

Show More
Commit Description:
handle the case when problem id or submission id is null. Grader will simply skip such request. Add more report on console (for command line grading)...
Commit Description:
handle the case when problem id or submission id is null. Grader will simply skip such request. Add more report on console (for command line grading) (mercurial grafted from d233105d3965c5368c9b33125f390e39b25f910e)
References:
r181:5c6f0b732e70 default
File last commit:
r120:7caf7717f327 default
Show/Diff file:
Action:
test/engine_spec.rb | 328 lines | 11.6 KiB | text/x-ruby | RubyLexer |
History | Source | Raw | Download
jittat
[grader] [MERGED] Merged new-arch-branch changes 74:105 into the trunk...
 r23 require File.join(File.dirname(__FILE__),'spec_helper')
require File.join(File.dirname(__FILE__),'engine_spec_helper')
describe "A grader engine, when grading submissions" do
include GraderEngineHelperMethods
before(:each) do
@config = Grader::Configuration.get_instance
# this test is from Pong
@problem_test_normal = stub(Problem,
:id => 1, :name => 'test_normal',
:full_score => 135)
@user_user1 = stub(User,
:id => 1, :login => 'user1')
@engine = Grader::Engine.new
init_sandbox
end
it "should grade normal submission" do
grader_should(:grade => "test1_correct.c",
:on => @problem_test_normal,
:and_report => {
:score => 135,
jittat
[grader] update spec...
 r65 :comment => /^(\[|P|\])+/})
jittat
[grader] [MERGED] Merged new-arch-branch changes 74:105 into the trunk...
 r23 end
Jittat Fakcharoenphol
added regression test for c++ submission
 r120 it "should grade normal submission in C++" do
cpp_lang = stub(Language, :name => 'cpp', :ext => 'cpp')
grader_should(:grade => "test1_correct_cc.cc",
:language => cpp_lang,
:on => @problem_test_normal,
:and_report => {
:score => 135,
:comment => /^(\[|P|\])+/})
end
jittat
[grader] [MERGED] Merged new-arch-branch changes 74:105 into the trunk...
 r23 it "should produce error message when submission cannot compile" do
grader_should(:grade => "test1_compile_error.c",
:on => @problem_test_normal,
:and_report => {
:score => 0,
jittat
[grader] update spec...
 r65 :comment => 'compilation error',
jittat
[grader] [MERGED] Merged new-arch-branch changes 74:105 into the trunk...
 r23 :compiler_message => /[Ee]rror/})
end
it "should produce timeout error when submission runs forever" do
@problem_test_timeout = stub(Problem,
:id => 1, :name => 'test_timeout',
:full_score => 10)
grader_should(:grade => "test2_timeout.c",
:on => @problem_test_timeout,
:and_report => {
:score => 0,
jittat
[grader] update spec...
 r65 :comment => 'TT'})
jittat
[grader] [MERGED] Merged new-arch-branch changes 74:105 into the trunk...
 r23 end
jittat
[grader] added fractional timelimit...
 r51 it "should produce timeout error correctly with fractional running time and fractional time limits" do
@problem_test_timeout = stub(Problem,
jittat
[grader] [MERGED] Merged new-arch-branch changes 74:105 into the trunk...
 r23 :id => 1, :name => 'test_timeout',
:full_score => 20)
jittat
[grader] added fractional timelimit...
 r51 grader_should(:grade => "test2_05sec.c",
jittat
[grader] [MERGED] Merged new-arch-branch changes 74:105 into the trunk...
 r23 :on => @problem_test_timeout,
:and_report => {
:score => 10,
jittat
[grader] update spec...
 r65 :comment => 'TP'})
jittat
[grader] [MERGED] Merged new-arch-branch changes 74:105 into the trunk...
 r23 end
it "should produce runtime error when submission uses too much static memory" do
@problem_test_memory = stub(Problem,
:id => 1, :name => 'test_memory',
:full_score => 20)
grader_should(:grade => "add_too_much_memory_static.c",
:on => @problem_test_memory,
:and_report => {
:score => 10,
jittat
[grader] update spec...
 r65 :comment => /[^P]P/})
jittat
[grader] [MERGED] Merged new-arch-branch changes 74:105 into the trunk...
 r23 end
it "should not allow submission to allocate too much dynamic memory" do
@problem_test_memory = stub(Problem,
:id => 1, :name => 'test_memory',
:full_score => 20)
grader_should(:grade => "add_too_much_memory_dynamic.c",
:on => @problem_test_memory,
:and_report => {
:score => 10,
jittat
[grader] update spec...
 r65 :comment => /[^P]P/})
jittat
[grader] [MERGED] Merged new-arch-branch changes 74:105 into the trunk...
 r23 end
jittat
fix test run scoring bug...
 r32 it "should score test runs correctly when submission fails in some test case" do
grader_should(:grade => "add_fail_test_case_1.c",
:on => @problem_test_normal,
:and_report => {
:score => 105,
jittat
[grader] update spec...
 r65 :comment => /^.*(-|x).*$/})
jittat
fix test run scoring bug...
 r32 end
jittat
[grader] [MERGED] Merged new-arch-branch changes 74:105 into the trunk...
 r23 it "should fail submission with non-zero exit status" do
jittat
[grader] finish engine spec on grading non-zero exit status...
 r33 grader_should(:grade => "add_nonzero_exit_status.c",
:on => @problem_test_normal,
:and_report => {
:score => 0,
jittat
[grader] update spec...
 r65 :comment => /^(\[|x|\])+$/})
jittat
[grader] [MERGED] Merged new-arch-branch changes 74:105 into the trunk...
 r23 end
jittat
[grader] new test on accessing PROBLEM_HOME and openning files...
 r35 it "should not allow malicious submission to see PROBLEM_HOME" do
problem_test_yesno = stub(Problem,
:id => 1, :name => 'test_yesno',
:full_score => 10)
grader_should(:grade => "yesno_access_problem_home.c",
:on => problem_test_yesno,
:and_report => {
:score => 0,
jittat
[grader] update spec...
 r65 :comment => /(-|x)/})
jittat
[grader] new test on accessing PROBLEM_HOME and openning files...
 r35 end
it "should not allow malicious submission to open files" do
problem_test_yesno = stub(Problem,
:id => 1, :name => 'test_yesno',
:full_score => 10)
grader_should(:grade => "yesno_open_file.c",
:on => problem_test_yesno,
:and_report => {
:score => 0,
jittat
[grader] update spec...
 r65 :comment => /(-|x)/})
jittat
[grader] new test on accessing PROBLEM_HOME and openning files...
 r35 end
jittat
[grader] [MERGED] Merged new-arch-branch changes 74:105 into the trunk...
 r23 def grader_should(args)
@user1 = stub(User,
:id => 1, :login => 'user1')
Jittat Fakcharoenphol
added regression test for c++ submission
 r120
jittat
[grader] [MERGED] Merged new-arch-branch changes 74:105 into the trunk...
 r23 submission =
Jittat Fakcharoenphol
added regression test for c++ submission
 r120 create_submission_from_file(1, @user1, args[:on], args[:grade], args[:language])
jittat
[grader] [MERGED] Merged new-arch-branch changes 74:105 into the trunk...
 r23 submission.should_receive(:graded_at=)
expected_score = args[:and_report][:score]
expected_comment = args[:and_report][:comment]
if args[:and_report][:compiler_message]!=nil
expected_compiler_message = args[:and_report][:compiler_message]
else
expected_compiler_message = ''
end
submission.should_receive(:points=).with(expected_score)
submission.should_receive(:grader_comment=).with(expected_comment)
submission.should_receive(:compiler_message=).with(expected_compiler_message)
submission.should_receive(:save)
@engine.grade(submission)
end
protected
def create_normal_submission_mock_from_file(source_fname)
create_submission_from_file(1, @user_user1, @problem_test_normal, source_fname)
end
end
describe "A grader engine, when grading test requests" do
include GraderEngineHelperMethods
before(:each) do
@config = Grader::Configuration.get_instance
Jittat Fakcharoenphol
added grading report
 r92 @engine = Grader::Engine.new(:room_maker => Grader::TestRequestRoomMaker.new,
:reporter => Grader::TestRequestReporter.new)
jittat
[grader] [MERGED] Merged new-arch-branch changes 74:105 into the trunk...
 r23 init_sandbox
end
it "should report error if there is no problem template" do
problem = stub(Problem,
:id => 1, :name => 'nothing')
grader_should(:grade => 'test1_correct.c',
:on => problem,
:with => 'in1.txt',
:and_report => {
:graded_at= => nil,
:compiler_message= => '',
:grader_comment= => '',
:running_stat= => /template not found/,
jittat
[grader] extracted test_request info into new stat columns...
 r41 :running_time= => nil,
:exit_status= => nil,
:memory_usage= => nil,
jittat
[grader] [MERGED] Merged new-arch-branch changes 74:105 into the trunk...
 r23 :save => nil})
end
it "should run test request and produce output file" do
problem = stub(Problem,
:id => 1, :name => 'test_normal')
grader_should(:grade => 'test1_correct.c',
:on => problem,
:with => 'in1.txt',
:and_report => {
:graded_at= => nil,
:compiler_message= => '',
:grader_comment= => '',
jittat
[grader] small tweak on one test case...
 r44 :running_stat= => /0.0\d* sec./,
jittat
[grader] [MERGED] Merged new-arch-branch changes 74:105 into the trunk...
 r23 :output_file_name= => lambda { |fname|
File.exists?(fname).should be_true
},
jittat
[grader] extracted test_request info into new stat columns...
 r41 :running_time= => nil,
:exit_status= => nil,
:memory_usage= => nil,
jittat
[grader] [MERGED] Merged new-arch-branch changes 74:105 into the trunk...
 r23 :save => nil})
end
it "should clean up problem directory after running test request" do
problem = stub(Problem,
:id => 1, :name => 'test_normal')
grader_should(:grade => 'test1_correct.c',
:on => problem,
:with => 'in1.txt',
:and_report => {
:graded_at= => nil,
:compiler_message= => '',
:grader_comment= => '',
:running_stat= => nil,
:output_file_name= => nil,
jittat
[grader] extracted test_request info into new stat columns...
 r41 :running_time= => nil,
:exit_status= => nil,
:memory_usage= => nil,
jittat
[grader] [MERGED] Merged new-arch-branch changes 74:105 into the trunk...
 r23 :save => nil})
File.exists?(@config.user_result_dir + "/test_request/test_normal/test_cases/1/input-1.txt").should be_false
end
it "should compile test request with error and report compilation error" do
problem = stub(Problem,
:id => 1, :name => 'test_normal')
grader_should(:grade => 'test1_compile_error.c',
:on => problem,
:with => 'in1.txt',
:and_report => {
:graded_at= => nil,
:compiler_message= => /.+/,
:grader_comment= => /[Cc]ompil.*error/,
:running_stat= => '',
:save => nil})
end
it "should report exit status" do
problem = stub(Problem,
:id => 1, :name => 'test_normal')
grader_should(:grade => 'add_nonzero_exit_status.c',
:on => problem,
:with => 'in1.txt',
:and_report => {
:graded_at= => nil,
:compiler_message= => '',
:grader_comment= => '',
jittat
[grader] box.cc now reports running status when die...
 r46 :running_stat= => /[Ee]xit.*status.*10.*0\.0\d* sec/m,
jittat
[grader] [MERGED] Merged new-arch-branch changes 74:105 into the trunk...
 r23 :output_file_name= => lambda { |fname|
File.exists?(fname).should be_true
},
jittat
[grader] extracted test_request info into new stat columns...
 r41 :running_time= => nil,
:exit_status= => /10/,
:memory_usage= => nil,
jittat
[grader] [MERGED] Merged new-arch-branch changes 74:105 into the trunk...
 r23 :save => nil})
end
jittat
[grader] added memory usage status report, better running time report...
 r42 it "should produce running statistics for normal submission" do
problem = stub(Problem,
:id => 1, :name => 'test_normal')
grader_should(:grade => 'test_run_stat.c',
:on => problem,
:with => 'in1.txt',
:and_report => {
:graded_at= => nil,
:compiler_message= => '',
:grader_comment= => '',
:running_stat= => nil,
:output_file_name= => lambda { |fname|
File.exists?(fname).should be_true
},
:running_time= => lambda { |t|
(t>=0.14) and (t<=0.16)
},
:exit_status= => nil,
:memory_usage= => lambda { |s|
(s>=500) and (s<=1000)
},
:save => nil})
end
jittat
[grader] [MERGED] Merged new-arch-branch changes 74:105 into the trunk...
 r23 protected
def grader_should(args)
@user1 = stub(User,
:id => 1, :login => 'user1')
problem = args[:on]
input_file = @config.test_request_input_base_dir + "/" + args[:with]
submission =
create_submission_from_file(1, @user1, args[:on], args[:grade])
test_request = stub(TestRequest,
:id => 1,
:user => @user1,
:problem => problem,
:submission => submission,
:input_file_name => input_file,
:language => submission.language,
:problem_name => problem.name)
expectations = args[:and_report]
expectations.each do |key,val|
if val==nil
test_request.should_receive(key)
elsif val.class == Proc
test_request.should_receive(key) { |fname|
val.call(fname)
}
else
test_request.should_receive(key).with(val)
end
end
@engine.grade(test_request)
end
end