cafe-grader-judge-scripts

Commit r120:7caf7717f327 public

parent | child
Description:
added regression test for c++ submission
Commit status:
[Not Reviewed]
References:
default
Diff options:
Raw Diff | Patch Diff | Download Diff | Ignore whitespace | Increase context
Comments:
0 Commit comments 0 Inline Comments
Unresolved TODOs:
There are no unresolved TODOs
Side by Side Unified
Expand All Files Collapse All Files Wide Mode Diff
Add another comment
Browse Files

r120:7caf7717f327 - - 2 files changed: 26 inserted, 1 deleted

Show file before | Show file after | Raw diff | Download diff | Ignore whitespace Increase context Show commentsHide comments
@@ -0,0 +1,13
1 + /*
2 + LANG: C++
3 + */
4 + #include <cstdio>
5 +
6 + int main()
7 + {
8 + int a,b,r;
9 + r = scanf("%d %d",&a,&b);
10 + printf("%d\n",a+b);
11 + return 0;
12 + }
13 +
Show file before | Show file after | Raw diff | Download diff | Ignore whitespace Increase context Show commentsHide comments
@@ -1,222 +1,234
1 1 require File.join(File.dirname(__FILE__),'spec_helper')
2 2 require File.join(File.dirname(__FILE__),'engine_spec_helper')
3 3
4 4 describe "A grader engine, when grading submissions" do
5 5
6 6 include GraderEngineHelperMethods
7 7
8 8 before(:each) do
9 9 @config = Grader::Configuration.get_instance
10 10
11 11 # this test is from Pong
12 12 @problem_test_normal = stub(Problem,
13 13 :id => 1, :name => 'test_normal',
14 14 :full_score => 135)
15 15 @user_user1 = stub(User,
16 16 :id => 1, :login => 'user1')
17 17
18 18 @engine = Grader::Engine.new
19 19 init_sandbox
20 20 end
21 21
22 22 it "should grade normal submission" do
23 23 grader_should(:grade => "test1_correct.c",
24 24 :on => @problem_test_normal,
25 25 :and_report => {
26 26 :score => 135,
27 27 :comment => /^(\[|P|\])+/})
28 28 end
29 29
30 30
31 + it "should grade normal submission in C++" do
32 + cpp_lang = stub(Language, :name => 'cpp', :ext => 'cpp')
33 +
34 + grader_should(:grade => "test1_correct_cc.cc",
35 + :language => cpp_lang,
36 + :on => @problem_test_normal,
37 + :and_report => {
38 + :score => 135,
39 + :comment => /^(\[|P|\])+/})
40 + end
41 +
31 42 it "should produce error message when submission cannot compile" do
32 43 grader_should(:grade => "test1_compile_error.c",
33 44 :on => @problem_test_normal,
34 45 :and_report => {
35 46 :score => 0,
36 47 :comment => 'compilation error',
37 48 :compiler_message => /[Ee]rror/})
38 49 end
39 50
40 51 it "should produce timeout error when submission runs forever" do
41 52 @problem_test_timeout = stub(Problem,
42 53 :id => 1, :name => 'test_timeout',
43 54 :full_score => 10)
44 55 grader_should(:grade => "test2_timeout.c",
45 56 :on => @problem_test_timeout,
46 57 :and_report => {
47 58 :score => 0,
48 59 :comment => 'TT'})
49 60 end
50 61
51 62 it "should produce timeout error correctly with fractional running time and fractional time limits" do
52 63 @problem_test_timeout = stub(Problem,
53 64 :id => 1, :name => 'test_timeout',
54 65 :full_score => 20)
55 66 grader_should(:grade => "test2_05sec.c",
56 67 :on => @problem_test_timeout,
57 68 :and_report => {
58 69 :score => 10,
59 70 :comment => 'TP'})
60 71 end
61 72
62 73 it "should produce runtime error when submission uses too much static memory" do
63 74 @problem_test_memory = stub(Problem,
64 75 :id => 1, :name => 'test_memory',
65 76 :full_score => 20)
66 77 grader_should(:grade => "add_too_much_memory_static.c",
67 78 :on => @problem_test_memory,
68 79 :and_report => {
69 80 :score => 10,
70 81 :comment => /[^P]P/})
71 82 end
72 83
73 84 it "should not allow submission to allocate too much dynamic memory" do
74 85 @problem_test_memory = stub(Problem,
75 86 :id => 1, :name => 'test_memory',
76 87 :full_score => 20)
77 88 grader_should(:grade => "add_too_much_memory_dynamic.c",
78 89 :on => @problem_test_memory,
79 90 :and_report => {
80 91 :score => 10,
81 92 :comment => /[^P]P/})
82 93 end
83 94
84 95 it "should score test runs correctly when submission fails in some test case" do
85 96 grader_should(:grade => "add_fail_test_case_1.c",
86 97 :on => @problem_test_normal,
87 98 :and_report => {
88 99 :score => 105,
89 100 :comment => /^.*(-|x).*$/})
90 101 end
91 102
92 103 it "should fail submission with non-zero exit status" do
93 104 grader_should(:grade => "add_nonzero_exit_status.c",
94 105 :on => @problem_test_normal,
95 106 :and_report => {
96 107 :score => 0,
97 108 :comment => /^(\[|x|\])+$/})
98 109 end
99 110
100 111 it "should not allow malicious submission to see PROBLEM_HOME" do
101 112 problem_test_yesno = stub(Problem,
102 113 :id => 1, :name => 'test_yesno',
103 114 :full_score => 10)
104 115 grader_should(:grade => "yesno_access_problem_home.c",
105 116 :on => problem_test_yesno,
106 117 :and_report => {
107 118 :score => 0,
108 119 :comment => /(-|x)/})
109 120 end
110 121
111 122 it "should not allow malicious submission to open files" do
112 123 problem_test_yesno = stub(Problem,
113 124 :id => 1, :name => 'test_yesno',
114 125 :full_score => 10)
115 126 grader_should(:grade => "yesno_open_file.c",
116 127 :on => problem_test_yesno,
117 128 :and_report => {
118 129 :score => 0,
119 130 :comment => /(-|x)/})
120 131 end
121 132
122 133 def grader_should(args)
123 134 @user1 = stub(User,
124 135 :id => 1, :login => 'user1')
136 +
125 137 submission =
126 - create_submission_from_file(1, @user1, args[:on], args[:grade])
138 + create_submission_from_file(1, @user1, args[:on], args[:grade], args[:language])
127 139 submission.should_receive(:graded_at=)
128 140
129 141 expected_score = args[:and_report][:score]
130 142 expected_comment = args[:and_report][:comment]
131 143 if args[:and_report][:compiler_message]!=nil
132 144 expected_compiler_message = args[:and_report][:compiler_message]
133 145 else
134 146 expected_compiler_message = ''
135 147 end
136 148
137 149 submission.should_receive(:points=).with(expected_score)
138 150 submission.should_receive(:grader_comment=).with(expected_comment)
139 151 submission.should_receive(:compiler_message=).with(expected_compiler_message)
140 152 submission.should_receive(:save)
141 153
142 154 @engine.grade(submission)
143 155 end
144 156
145 157 protected
146 158
147 159 def create_normal_submission_mock_from_file(source_fname)
148 160 create_submission_from_file(1, @user_user1, @problem_test_normal, source_fname)
149 161 end
150 162
151 163 end
152 164
153 165 describe "A grader engine, when grading test requests" do
154 166
155 167 include GraderEngineHelperMethods
156 168
157 169 before(:each) do
158 170 @config = Grader::Configuration.get_instance
159 171 @engine = Grader::Engine.new(:room_maker => Grader::TestRequestRoomMaker.new,
160 172 :reporter => Grader::TestRequestReporter.new)
161 173 init_sandbox
162 174 end
163 175
164 176 it "should report error if there is no problem template" do
165 177 problem = stub(Problem,
166 178 :id => 1, :name => 'nothing')
167 179 grader_should(:grade => 'test1_correct.c',
168 180 :on => problem,
169 181 :with => 'in1.txt',
170 182 :and_report => {
171 183 :graded_at= => nil,
172 184 :compiler_message= => '',
173 185 :grader_comment= => '',
174 186 :running_stat= => /template not found/,
175 187 :running_time= => nil,
176 188 :exit_status= => nil,
177 189 :memory_usage= => nil,
178 190 :save => nil})
179 191 end
180 192
181 193 it "should run test request and produce output file" do
182 194 problem = stub(Problem,
183 195 :id => 1, :name => 'test_normal')
184 196 grader_should(:grade => 'test1_correct.c',
185 197 :on => problem,
186 198 :with => 'in1.txt',
187 199 :and_report => {
188 200 :graded_at= => nil,
189 201 :compiler_message= => '',
190 202 :grader_comment= => '',
191 203 :running_stat= => /0.0\d* sec./,
192 204 :output_file_name= => lambda { |fname|
193 205 File.exists?(fname).should be_true
194 206 },
195 207 :running_time= => nil,
196 208 :exit_status= => nil,
197 209 :memory_usage= => nil,
198 210 :save => nil})
199 211 end
200 212
201 213 it "should clean up problem directory after running test request" do
202 214 problem = stub(Problem,
203 215 :id => 1, :name => 'test_normal')
204 216 grader_should(:grade => 'test1_correct.c',
205 217 :on => problem,
206 218 :with => 'in1.txt',
207 219 :and_report => {
208 220 :graded_at= => nil,
209 221 :compiler_message= => '',
210 222 :grader_comment= => '',
211 223 :running_stat= => nil,
212 224 :output_file_name= => nil,
213 225 :running_time= => nil,
214 226 :exit_status= => nil,
215 227 :memory_usage= => nil,
216 228 :save => nil})
217 229 File.exists?(@config.user_result_dir + "/test_request/test_normal/test_cases/1/input-1.txt").should be_false
218 230 end
219 231
220 232 it "should compile test request with error and report compilation error" do
221 233 problem = stub(Problem,
222 234 :id => 1, :name => 'test_normal')
You need to be logged in to leave comments. Login now