cafe-grader-judge-scripts

Commit r120:7caf7717f327 public

parent | child
Description:
added regression test for c++ submission
Commit status:
[Not Reviewed]
References:
default
Diff options:
Raw Diff | Patch Diff | Download Diff | Ignore whitespace | Increase context
Comments:
0 Commit comments 0 Inline Comments
Unresolved TODOs:
There are no unresolved TODOs
Side by Side Unified
Expand All Files Collapse All Files Wide Mode Diff
Add another comment
Browse Files

r120:7caf7717f327 - - 2 files changed: 26 inserted, 1 deleted

Show file before | Show file after | Raw diff | Download diff | Ignore whitespace Increase context Show commentsHide comments
@@ -0,0 +1,13
1 + /*
2 + LANG: C++
3 + */
4 + #include <cstdio>
5 +
6 + int main()
7 + {
8 + int a,b,r;
9 + r = scanf("%d %d",&a,&b);
10 + printf("%d\n",a+b);
11 + return 0;
12 + }
13 +
Show file before | Show file after | Raw diff | Download diff | Ignore whitespace Increase context Show commentsHide comments
@@ -1,174 +1,186
1 require File.join(File.dirname(__FILE__),'spec_helper')
1 require File.join(File.dirname(__FILE__),'spec_helper')
2 require File.join(File.dirname(__FILE__),'engine_spec_helper')
2 require File.join(File.dirname(__FILE__),'engine_spec_helper')
3
3
4 describe "A grader engine, when grading submissions" do
4 describe "A grader engine, when grading submissions" do
5
5
6 include GraderEngineHelperMethods
6 include GraderEngineHelperMethods
7
7
8 before(:each) do
8 before(:each) do
9 @config = Grader::Configuration.get_instance
9 @config = Grader::Configuration.get_instance
10
10
11 # this test is from Pong
11 # this test is from Pong
12 @problem_test_normal = stub(Problem,
12 @problem_test_normal = stub(Problem,
13 :id => 1, :name => 'test_normal',
13 :id => 1, :name => 'test_normal',
14 :full_score => 135)
14 :full_score => 135)
15 @user_user1 = stub(User,
15 @user_user1 = stub(User,
16 :id => 1, :login => 'user1')
16 :id => 1, :login => 'user1')
17
17
18 @engine = Grader::Engine.new
18 @engine = Grader::Engine.new
19 init_sandbox
19 init_sandbox
20 end
20 end
21
21
22 it "should grade normal submission" do
22 it "should grade normal submission" do
23 grader_should(:grade => "test1_correct.c",
23 grader_should(:grade => "test1_correct.c",
24 :on => @problem_test_normal,
24 :on => @problem_test_normal,
25 :and_report => {
25 :and_report => {
26 :score => 135,
26 :score => 135,
27 :comment => /^(\[|P|\])+/})
27 :comment => /^(\[|P|\])+/})
28 end
28 end
29
29
30
30
31 + it "should grade normal submission in C++" do
32 + cpp_lang = stub(Language, :name => 'cpp', :ext => 'cpp')
33 +
34 + grader_should(:grade => "test1_correct_cc.cc",
35 + :language => cpp_lang,
36 + :on => @problem_test_normal,
37 + :and_report => {
38 + :score => 135,
39 + :comment => /^(\[|P|\])+/})
40 + end
41 +
31 it "should produce error message when submission cannot compile" do
42 it "should produce error message when submission cannot compile" do
32 grader_should(:grade => "test1_compile_error.c",
43 grader_should(:grade => "test1_compile_error.c",
33 :on => @problem_test_normal,
44 :on => @problem_test_normal,
34 :and_report => {
45 :and_report => {
35 :score => 0,
46 :score => 0,
36 :comment => 'compilation error',
47 :comment => 'compilation error',
37 :compiler_message => /[Ee]rror/})
48 :compiler_message => /[Ee]rror/})
38 end
49 end
39
50
40 it "should produce timeout error when submission runs forever" do
51 it "should produce timeout error when submission runs forever" do
41 @problem_test_timeout = stub(Problem,
52 @problem_test_timeout = stub(Problem,
42 :id => 1, :name => 'test_timeout',
53 :id => 1, :name => 'test_timeout',
43 :full_score => 10)
54 :full_score => 10)
44 grader_should(:grade => "test2_timeout.c",
55 grader_should(:grade => "test2_timeout.c",
45 :on => @problem_test_timeout,
56 :on => @problem_test_timeout,
46 :and_report => {
57 :and_report => {
47 :score => 0,
58 :score => 0,
48 :comment => 'TT'})
59 :comment => 'TT'})
49 end
60 end
50
61
51 it "should produce timeout error correctly with fractional running time and fractional time limits" do
62 it "should produce timeout error correctly with fractional running time and fractional time limits" do
52 @problem_test_timeout = stub(Problem,
63 @problem_test_timeout = stub(Problem,
53 :id => 1, :name => 'test_timeout',
64 :id => 1, :name => 'test_timeout',
54 :full_score => 20)
65 :full_score => 20)
55 grader_should(:grade => "test2_05sec.c",
66 grader_should(:grade => "test2_05sec.c",
56 :on => @problem_test_timeout,
67 :on => @problem_test_timeout,
57 :and_report => {
68 :and_report => {
58 :score => 10,
69 :score => 10,
59 :comment => 'TP'})
70 :comment => 'TP'})
60 end
71 end
61
72
62 it "should produce runtime error when submission uses too much static memory" do
73 it "should produce runtime error when submission uses too much static memory" do
63 @problem_test_memory = stub(Problem,
74 @problem_test_memory = stub(Problem,
64 :id => 1, :name => 'test_memory',
75 :id => 1, :name => 'test_memory',
65 :full_score => 20)
76 :full_score => 20)
66 grader_should(:grade => "add_too_much_memory_static.c",
77 grader_should(:grade => "add_too_much_memory_static.c",
67 :on => @problem_test_memory,
78 :on => @problem_test_memory,
68 :and_report => {
79 :and_report => {
69 :score => 10,
80 :score => 10,
70 :comment => /[^P]P/})
81 :comment => /[^P]P/})
71 end
82 end
72
83
73 it "should not allow submission to allocate too much dynamic memory" do
84 it "should not allow submission to allocate too much dynamic memory" do
74 @problem_test_memory = stub(Problem,
85 @problem_test_memory = stub(Problem,
75 :id => 1, :name => 'test_memory',
86 :id => 1, :name => 'test_memory',
76 :full_score => 20)
87 :full_score => 20)
77 grader_should(:grade => "add_too_much_memory_dynamic.c",
88 grader_should(:grade => "add_too_much_memory_dynamic.c",
78 :on => @problem_test_memory,
89 :on => @problem_test_memory,
79 :and_report => {
90 :and_report => {
80 :score => 10,
91 :score => 10,
81 :comment => /[^P]P/})
92 :comment => /[^P]P/})
82 end
93 end
83
94
84 it "should score test runs correctly when submission fails in some test case" do
95 it "should score test runs correctly when submission fails in some test case" do
85 grader_should(:grade => "add_fail_test_case_1.c",
96 grader_should(:grade => "add_fail_test_case_1.c",
86 :on => @problem_test_normal,
97 :on => @problem_test_normal,
87 :and_report => {
98 :and_report => {
88 :score => 105,
99 :score => 105,
89 :comment => /^.*(-|x).*$/})
100 :comment => /^.*(-|x).*$/})
90 end
101 end
91
102
92 it "should fail submission with non-zero exit status" do
103 it "should fail submission with non-zero exit status" do
93 grader_should(:grade => "add_nonzero_exit_status.c",
104 grader_should(:grade => "add_nonzero_exit_status.c",
94 :on => @problem_test_normal,
105 :on => @problem_test_normal,
95 :and_report => {
106 :and_report => {
96 :score => 0,
107 :score => 0,
97 :comment => /^(\[|x|\])+$/})
108 :comment => /^(\[|x|\])+$/})
98 end
109 end
99
110
100 it "should not allow malicious submission to see PROBLEM_HOME" do
111 it "should not allow malicious submission to see PROBLEM_HOME" do
101 problem_test_yesno = stub(Problem,
112 problem_test_yesno = stub(Problem,
102 :id => 1, :name => 'test_yesno',
113 :id => 1, :name => 'test_yesno',
103 :full_score => 10)
114 :full_score => 10)
104 grader_should(:grade => "yesno_access_problem_home.c",
115 grader_should(:grade => "yesno_access_problem_home.c",
105 :on => problem_test_yesno,
116 :on => problem_test_yesno,
106 :and_report => {
117 :and_report => {
107 :score => 0,
118 :score => 0,
108 :comment => /(-|x)/})
119 :comment => /(-|x)/})
109 end
120 end
110
121
111 it "should not allow malicious submission to open files" do
122 it "should not allow malicious submission to open files" do
112 problem_test_yesno = stub(Problem,
123 problem_test_yesno = stub(Problem,
113 :id => 1, :name => 'test_yesno',
124 :id => 1, :name => 'test_yesno',
114 :full_score => 10)
125 :full_score => 10)
115 grader_should(:grade => "yesno_open_file.c",
126 grader_should(:grade => "yesno_open_file.c",
116 :on => problem_test_yesno,
127 :on => problem_test_yesno,
117 :and_report => {
128 :and_report => {
118 :score => 0,
129 :score => 0,
119 :comment => /(-|x)/})
130 :comment => /(-|x)/})
120 end
131 end
121
132
122 def grader_should(args)
133 def grader_should(args)
123 @user1 = stub(User,
134 @user1 = stub(User,
124 :id => 1, :login => 'user1')
135 :id => 1, :login => 'user1')
136 +
125 submission =
137 submission =
126 - create_submission_from_file(1, @user1, args[:on], args[:grade])
138 + create_submission_from_file(1, @user1, args[:on], args[:grade], args[:language])
127 submission.should_receive(:graded_at=)
139 submission.should_receive(:graded_at=)
128
140
129 expected_score = args[:and_report][:score]
141 expected_score = args[:and_report][:score]
130 expected_comment = args[:and_report][:comment]
142 expected_comment = args[:and_report][:comment]
131 if args[:and_report][:compiler_message]!=nil
143 if args[:and_report][:compiler_message]!=nil
132 expected_compiler_message = args[:and_report][:compiler_message]
144 expected_compiler_message = args[:and_report][:compiler_message]
133 else
145 else
134 expected_compiler_message = ''
146 expected_compiler_message = ''
135 end
147 end
136
148
137 submission.should_receive(:points=).with(expected_score)
149 submission.should_receive(:points=).with(expected_score)
138 submission.should_receive(:grader_comment=).with(expected_comment)
150 submission.should_receive(:grader_comment=).with(expected_comment)
139 submission.should_receive(:compiler_message=).with(expected_compiler_message)
151 submission.should_receive(:compiler_message=).with(expected_compiler_message)
140 submission.should_receive(:save)
152 submission.should_receive(:save)
141
153
142 @engine.grade(submission)
154 @engine.grade(submission)
143 end
155 end
144
156
145 protected
157 protected
146
158
147 def create_normal_submission_mock_from_file(source_fname)
159 def create_normal_submission_mock_from_file(source_fname)
148 create_submission_from_file(1, @user_user1, @problem_test_normal, source_fname)
160 create_submission_from_file(1, @user_user1, @problem_test_normal, source_fname)
149 end
161 end
150
162
151 end
163 end
152
164
153 describe "A grader engine, when grading test requests" do
165 describe "A grader engine, when grading test requests" do
154
166
155 include GraderEngineHelperMethods
167 include GraderEngineHelperMethods
156
168
157 before(:each) do
169 before(:each) do
158 @config = Grader::Configuration.get_instance
170 @config = Grader::Configuration.get_instance
159 @engine = Grader::Engine.new(:room_maker => Grader::TestRequestRoomMaker.new,
171 @engine = Grader::Engine.new(:room_maker => Grader::TestRequestRoomMaker.new,
160 :reporter => Grader::TestRequestReporter.new)
172 :reporter => Grader::TestRequestReporter.new)
161 init_sandbox
173 init_sandbox
162 end
174 end
163
175
164 it "should report error if there is no problem template" do
176 it "should report error if there is no problem template" do
165 problem = stub(Problem,
177 problem = stub(Problem,
166 :id => 1, :name => 'nothing')
178 :id => 1, :name => 'nothing')
167 grader_should(:grade => 'test1_correct.c',
179 grader_should(:grade => 'test1_correct.c',
168 :on => problem,
180 :on => problem,
169 :with => 'in1.txt',
181 :with => 'in1.txt',
170 :and_report => {
182 :and_report => {
171 :graded_at= => nil,
183 :graded_at= => nil,
172 :compiler_message= => '',
184 :compiler_message= => '',
173 :grader_comment= => '',
185 :grader_comment= => '',
174 :running_stat= => /template not found/,
186 :running_stat= => /template not found/,
You need to be logged in to leave comments. Login now