cafe-grader-judge-scripts

Commit r33:e7a6930e8c87 public

parent | child
Description:
[grader] finish engine spec on grading non-zero exit status git-svn-id: http://theory.cpe.ku.ac.th/grader/judge/trunk/scripts@127 6386c4cd-e34a-4fa8-8920-d93eb39b512e
Commit status:
[Not Reviewed]
References:
default
Diff options:
Raw Diff | Patch Diff | Download Diff | Ignore whitespace | Increase context
Comments:
0 Commit comments 0 Inline Comments
Unresolved TODOs:
There are no unresolved TODOs
Side by Side Unified
Expand All Files Collapse All Files Wide Mode Diff
Add another comment
Browse Files

r33:e7a6930e8c87 - - 1 file changed: 5 inserted, 1 deleted

Show file before | Show file after | Raw diff | Download diff | Ignore whitespace Increase context Show commentsHide comments
@@ -45,97 +45,101
45 :on => @problem_test_timeout,
45 :on => @problem_test_timeout,
46 :and_report => {
46 :and_report => {
47 :score => 0,
47 :score => 0,
48 :comment => 'FAILED: TT'})
48 :comment => 'FAILED: TT'})
49 end
49 end
50
50
51 it "should produce timeout error correctly when submission runs slower than expected in less than a second" do
51 it "should produce timeout error correctly when submission runs slower than expected in less than a second" do
52 @problem_test_timeout = stub(Problem,
52 @problem_test_timeout = stub(Problem,
53 :id => 1, :name => 'test_timeout',
53 :id => 1, :name => 'test_timeout',
54 :full_score => 20)
54 :full_score => 20)
55 grader_should(:grade => "test2_1-5sec.c",
55 grader_should(:grade => "test2_1-5sec.c",
56 :on => @problem_test_timeout,
56 :on => @problem_test_timeout,
57 :and_report => {
57 :and_report => {
58 :score => 10,
58 :score => 10,
59 :comment => 'FAILED: TP'})
59 :comment => 'FAILED: TP'})
60 end
60 end
61
61
62 it "should produce runtime error when submission uses too much static memory" do
62 it "should produce runtime error when submission uses too much static memory" do
63 @problem_test_memory = stub(Problem,
63 @problem_test_memory = stub(Problem,
64 :id => 1, :name => 'test_memory',
64 :id => 1, :name => 'test_memory',
65 :full_score => 20)
65 :full_score => 20)
66 grader_should(:grade => "add_too_much_memory_static.c",
66 grader_should(:grade => "add_too_much_memory_static.c",
67 :on => @problem_test_memory,
67 :on => @problem_test_memory,
68 :and_report => {
68 :and_report => {
69 :score => 10,
69 :score => 10,
70 :comment => /FAILED: [^P]P/})
70 :comment => /FAILED: [^P]P/})
71 end
71 end
72
72
73 it "should not allow submission to allocate too much dynamic memory" do
73 it "should not allow submission to allocate too much dynamic memory" do
74 @problem_test_memory = stub(Problem,
74 @problem_test_memory = stub(Problem,
75 :id => 1, :name => 'test_memory',
75 :id => 1, :name => 'test_memory',
76 :full_score => 20)
76 :full_score => 20)
77 grader_should(:grade => "add_too_much_memory_dynamic.c",
77 grader_should(:grade => "add_too_much_memory_dynamic.c",
78 :on => @problem_test_memory,
78 :on => @problem_test_memory,
79 :and_report => {
79 :and_report => {
80 :score => 10,
80 :score => 10,
81 :comment => /FAILED: [^P]P/})
81 :comment => /FAILED: [^P]P/})
82 end
82 end
83
83
84 it "should score test runs correctly when submission fails in some test case" do
84 it "should score test runs correctly when submission fails in some test case" do
85 grader_should(:grade => "add_fail_test_case_1.c",
85 grader_should(:grade => "add_fail_test_case_1.c",
86 :on => @problem_test_normal,
86 :on => @problem_test_normal,
87 :and_report => {
87 :and_report => {
88 :score => 105,
88 :score => 105,
89 :comment => /^FAILED:/})
89 :comment => /^FAILED:/})
90 end
90 end
91
91
92 it "should fail submission with non-zero exit status" do
92 it "should fail submission with non-zero exit status" do
93 - violated("has not been implemented")
93 + grader_should(:grade => "add_nonzero_exit_status.c",
94 + :on => @problem_test_normal,
95 + :and_report => {
96 + :score => 0,
97 + :comment => /^FAILED:/})
94 end
98 end
95
99
96 def grader_should(args)
100 def grader_should(args)
97 @user1 = stub(User,
101 @user1 = stub(User,
98 :id => 1, :login => 'user1')
102 :id => 1, :login => 'user1')
99 submission =
103 submission =
100 create_submission_from_file(1, @user1, args[:on], args[:grade])
104 create_submission_from_file(1, @user1, args[:on], args[:grade])
101 submission.should_receive(:graded_at=)
105 submission.should_receive(:graded_at=)
102
106
103 expected_score = args[:and_report][:score]
107 expected_score = args[:and_report][:score]
104 expected_comment = args[:and_report][:comment]
108 expected_comment = args[:and_report][:comment]
105 if args[:and_report][:compiler_message]!=nil
109 if args[:and_report][:compiler_message]!=nil
106 expected_compiler_message = args[:and_report][:compiler_message]
110 expected_compiler_message = args[:and_report][:compiler_message]
107 else
111 else
108 expected_compiler_message = ''
112 expected_compiler_message = ''
109 end
113 end
110
114
111 submission.should_receive(:points=).with(expected_score)
115 submission.should_receive(:points=).with(expected_score)
112 submission.should_receive(:grader_comment=).with(expected_comment)
116 submission.should_receive(:grader_comment=).with(expected_comment)
113 submission.should_receive(:compiler_message=).with(expected_compiler_message)
117 submission.should_receive(:compiler_message=).with(expected_compiler_message)
114 submission.should_receive(:save)
118 submission.should_receive(:save)
115
119
116 @engine.grade(submission)
120 @engine.grade(submission)
117 end
121 end
118
122
119 protected
123 protected
120
124
121 def create_normal_submission_mock_from_file(source_fname)
125 def create_normal_submission_mock_from_file(source_fname)
122 create_submission_from_file(1, @user_user1, @problem_test_normal, source_fname)
126 create_submission_from_file(1, @user_user1, @problem_test_normal, source_fname)
123 end
127 end
124
128
125 end
129 end
126
130
127 describe "A grader engine, when grading test requests" do
131 describe "A grader engine, when grading test requests" do
128
132
129 include GraderEngineHelperMethods
133 include GraderEngineHelperMethods
130
134
131 before(:each) do
135 before(:each) do
132 @config = Grader::Configuration.get_instance
136 @config = Grader::Configuration.get_instance
133 @engine = Grader::Engine.new(Grader::TestRequestRoomMaker.new,
137 @engine = Grader::Engine.new(Grader::TestRequestRoomMaker.new,
134 Grader::TestRequestReporter.new)
138 Grader::TestRequestReporter.new)
135 init_sandbox
139 init_sandbox
136 end
140 end
137
141
138 it "should report error if there is no problem template" do
142 it "should report error if there is no problem template" do
139 problem = stub(Problem,
143 problem = stub(Problem,
140 :id => 1, :name => 'nothing')
144 :id => 1, :name => 'nothing')
141 grader_should(:grade => 'test1_correct.c',
145 grader_should(:grade => 'test1_correct.c',
You need to be logged in to leave comments. Login now