cafe-grader-judge-scripts

Commit r33:e7a6930e8c87 public

parent | child
Description:
[grader] finish engine spec on grading non-zero exit status git-svn-id: http://theory.cpe.ku.ac.th/grader/judge/trunk/scripts@127 6386c4cd-e34a-4fa8-8920-d93eb39b512e
Commit status:
[Not Reviewed]
References:
default
Diff options:
Raw Diff | Patch Diff | Download Diff | Show whitespace | Increase context
Comments:
0 Commit comments 0 Inline Comments
Unresolved TODOs:
There are no unresolved TODOs
Side by Side Unified
Expand All Files Collapse All Files Wide Mode Diff
Add another comment
Browse Files

r33:e7a6930e8c87 - - 1 file changed: 5 inserted, 1 deleted

Show file before | Show file after | Raw diff | Download diff | Show whitespace Increase context Show commentsHide comments
@@ -45,97 +45,101
45 45 :on => @problem_test_timeout,
46 46 :and_report => {
47 47 :score => 0,
48 48 :comment => 'FAILED: TT'})
49 49 end
50 50
51 51 it "should produce timeout error correctly when submission runs slower than expected in less than a second" do
52 52 @problem_test_timeout = stub(Problem,
53 53 :id => 1, :name => 'test_timeout',
54 54 :full_score => 20)
55 55 grader_should(:grade => "test2_1-5sec.c",
56 56 :on => @problem_test_timeout,
57 57 :and_report => {
58 58 :score => 10,
59 59 :comment => 'FAILED: TP'})
60 60 end
61 61
62 62 it "should produce runtime error when submission uses too much static memory" do
63 63 @problem_test_memory = stub(Problem,
64 64 :id => 1, :name => 'test_memory',
65 65 :full_score => 20)
66 66 grader_should(:grade => "add_too_much_memory_static.c",
67 67 :on => @problem_test_memory,
68 68 :and_report => {
69 69 :score => 10,
70 70 :comment => /FAILED: [^P]P/})
71 71 end
72 72
73 73 it "should not allow submission to allocate too much dynamic memory" do
74 74 @problem_test_memory = stub(Problem,
75 75 :id => 1, :name => 'test_memory',
76 76 :full_score => 20)
77 77 grader_should(:grade => "add_too_much_memory_dynamic.c",
78 78 :on => @problem_test_memory,
79 79 :and_report => {
80 80 :score => 10,
81 81 :comment => /FAILED: [^P]P/})
82 82 end
83 83
84 84 it "should score test runs correctly when submission fails in some test case" do
85 85 grader_should(:grade => "add_fail_test_case_1.c",
86 86 :on => @problem_test_normal,
87 87 :and_report => {
88 88 :score => 105,
89 89 :comment => /^FAILED:/})
90 90 end
91 91
92 92 it "should fail submission with non-zero exit status" do
93 - violated("has not been implemented")
93 + grader_should(:grade => "add_nonzero_exit_status.c",
94 + :on => @problem_test_normal,
95 + :and_report => {
96 + :score => 0,
97 + :comment => /^FAILED:/})
94 98 end
95 99
96 100 def grader_should(args)
97 101 @user1 = stub(User,
98 102 :id => 1, :login => 'user1')
99 103 submission =
100 104 create_submission_from_file(1, @user1, args[:on], args[:grade])
101 105 submission.should_receive(:graded_at=)
102 106
103 107 expected_score = args[:and_report][:score]
104 108 expected_comment = args[:and_report][:comment]
105 109 if args[:and_report][:compiler_message]!=nil
106 110 expected_compiler_message = args[:and_report][:compiler_message]
107 111 else
108 112 expected_compiler_message = ''
109 113 end
110 114
111 115 submission.should_receive(:points=).with(expected_score)
112 116 submission.should_receive(:grader_comment=).with(expected_comment)
113 117 submission.should_receive(:compiler_message=).with(expected_compiler_message)
114 118 submission.should_receive(:save)
115 119
116 120 @engine.grade(submission)
117 121 end
118 122
119 123 protected
120 124
121 125 def create_normal_submission_mock_from_file(source_fname)
122 126 create_submission_from_file(1, @user_user1, @problem_test_normal, source_fname)
123 127 end
124 128
125 129 end
126 130
127 131 describe "A grader engine, when grading test requests" do
128 132
129 133 include GraderEngineHelperMethods
130 134
131 135 before(:each) do
132 136 @config = Grader::Configuration.get_instance
133 137 @engine = Grader::Engine.new(Grader::TestRequestRoomMaker.new,
134 138 Grader::TestRequestReporter.new)
135 139 init_sandbox
136 140 end
137 141
138 142 it "should report error if there is no problem template" do
139 143 problem = stub(Problem,
140 144 :id => 1, :name => 'nothing')
141 145 grader_should(:grade => 'test1_correct.c',
You need to be logged in to leave comments. Login now