cafe-grader-judge-scripts

Commit r37:effd16a52c43 public

parent | child
Description:
[grader] fixed bug #6 and #9 git-svn-id: http://theory.cpe.ku.ac.th/grader/judge/trunk/scripts@139 6386c4cd-e34a-4fa8-8920-d93eb39b512e
Commit status:
[Not Reviewed]
References:
default
Diff options:
Raw Diff | Patch Diff | Download Diff | Ignore whitespace | Increase context
Comments:
0 Commit comments 0 Inline Comments
Unresolved TODOs:
There are no unresolved TODOs
Side by Side Unified
Expand All Files Collapse All Files Wide Mode Diff
Add another comment
Browse Files

r37:effd16a52c43 - - 5 files changed: 13 inserted, 3 deleted

Show file before | Show file after | Raw diff | Download diff | Ignore whitespace Increase context Show commentsHide comments
@@ -1,82 +1,84
1 1 module Grader
2 2
3 3 # This singleton class holds basic configurations for grader. When
4 4 # running in each mode, grader uses resources from different
5 5 # directories and outputs differently. Usually the attributes name
6 6 # are descriptive; below we explain more on each attributes.
7 7 class Configuration
8 8 # Rails' environment: "development", "production"
9 9 attr_accessor :rails_env
10 10
11 11 # Grader looks for problem [prob] in problem_dir/[prob], and store
12 12 # execution results for submission [x] of user [u] in directory
13 13 # user_result_dir/[u]/[x]
14 14 attr_accessor :problems_dir
15 15 attr_accessor :user_result_dir
16 16
17 17 # If report_grader=true, the grader would add a row in model
18 18 # GraderProcess. It would report itself with grader_hostname and
19 19 # process id.
20 20 attr_accessor :report_grader
21 21 attr_accessor :grader_hostname
22 22
23 23 # If talkative=true, grader would report status to console. If
24 24 # logging=true, grader would report status to a log file located
25 25 # in log_dir, in a file name mode.options.pid. TODO: defined
26 26 # log file naming.
27 27 attr_accessor :talkative
28 28 attr_accessor :logging
29 29 attr_accessor :log_dir
30 30
31 31 # These are directories related to the test interface.
32 32 attr_accessor :test_request_input_base_dir
33 33 attr_accessor :test_request_output_base_dir
34 34 attr_accessor :test_request_problem_templates_dir
35 35
36 36 # Comment received from the grading script will be filtered
37 37 # through Configuration#report_comment. How this method behave
38 38 # depends on this option; right now only two formats, :short and
39 39 # :long
40 40 attr_accessor :comment_report_style
41 41
42 42 def report_comment(comment)
43 43 case comment_report_style
44 44 when :short
45 - if comment.chomp =~ /^P+$/ # all P's
45 + if comment.chomp =~ /^[\[\]P]+$/ # all P's
46 46 'passed'
47 + elsif comment.chomp =~ /[Cc]ompil.*[Ee]rror/
48 + 'compilation error'
47 49 else
48 50 'failed'
49 51 end
50 52
51 53 when :full
52 54 comment.chomp
53 55 end
54 56 end
55 57
56 58 # Codes for singleton
57 59 private_class_method :new
58 60
59 61 @@instance = nil
60 62
61 63 def self.get_instance
62 64 if @@instance==nil
63 65 @@instance = new
64 66 end
65 67 @@instance
66 68 end
67 69
68 70 private
69 71 def initialize
70 72 @talkative = false
71 73 @log_file = nil
72 74 @report_grader = false
73 75 @grader_hostname = `hostname`.chomp
74 76
75 77 @rails_env = 'development'
76 78
77 79 @comment_report_style = :full
78 80 end
79 81
80 82 end
81 83
82 84 end
Show file before | Show file after | Raw diff | Download diff | Ignore whitespace Increase context Show commentsHide comments
@@ -1,115 +1,117
1 1 module Grader
2 2
3 3 class SubmissionRoomMaker
4 4 def initialize
5 5 @config = Grader::Configuration.get_instance
6 6 end
7 7
8 8 def produce_grading_room(submission)
9 9 user = submission.user
10 10 problem = submission.problem
11 11 grading_room = "#{@config.user_result_dir}/" +
12 12 "#{user.login}/#{problem.name}/#{submission.id}"
13 13
14 14 FileUtils.mkdir_p(grading_room)
15 15 grading_room
16 16 end
17 17
18 18 def find_problem_home(submission)
19 19 problem = submission.problem
20 20 "#{@config.problems_dir}/#{problem.name}"
21 21 end
22 22
23 23 def save_source(submission,source_name)
24 24 dir = self.produce_grading_room(submission)
25 25 f = File.open("#{dir}/#{source_name}","w")
26 26 f.write(submission.source)
27 27 f.close
28 28 end
29 29
30 30 def clean_up(submission)
31 31 end
32 32 end
33 33
34 34 class SubmissionReporter
35 35 def initialize
36 36 @config = Grader::Configuration.get_instance
37 37 end
38 38
39 39 def report(sub,test_result_dir)
40 40 save_result(sub,read_result(test_result_dir))
41 41 end
42 42
43 43 def report_error(sub,msg)
44 44 save_result(sub,{:points => 0,
45 45 :comment => "Grading error: #{msg}" })
46 46 end
47 47
48 48 protected
49 49 def read_result(test_result_dir)
50 50 cmp_msg_fname = "#{test_result_dir}/compiler_message"
51 51 if FileTest.exist?(cmp_msg_fname)
52 52 cmp_file = File.open(cmp_msg_fname)
53 53 cmp_msg = cmp_file.read
54 54 cmp_file.close
55 55 else
56 56 cmp_msg = ""
57 57 end
58 58
59 59 result_fname = "#{test_result_dir}/result"
60 60 comment_fname = "#{test_result_dir}/comment"
61 61 if FileTest.exist?(result_fname)
62 62 comment = ""
63 63 begin
64 64 result_file = File.open(result_fname)
65 65 result = result_file.readline.to_i
66 66 result_file.close
67 67 rescue
68 68 result = 0
69 69 comment = "error reading result file."
70 70 end
71 71
72 72 begin
73 73 comment_file = File.open(comment_fname)
74 74 comment += comment_file.readline.chomp
75 75 comment_file.close
76 76 rescue
77 77 comment += ""
78 78 end
79 79
80 80 return {:points => result,
81 81 :comment => comment,
82 82 :cmp_msg => cmp_msg}
83 83 else
84 84 if FileTest.exist?("#{test_result_dir}/a.out")
85 85 return {:points => 0,
86 86 :comment => 'error during grading',
87 87 :cmp_msg => cmp_msg}
88 88 else
89 89 return {:points => 0,
90 - :comment => 'compile error',
90 + :comment => 'compilation error',
91 91 :cmp_msg => cmp_msg}
92 92 end
93 93 end
94 94 end
95 95
96 96 def save_result(submission,result)
97 97 problem = submission.problem
98 98 submission.graded_at = Time.now
99 99 points = result[:points]
100 100 submission.points = points
101 101 comment = @config.report_comment(result[:comment])
102 102 if problem == nil
103 103 submission.grader_comment = 'PASSED: ' + comment + '(problem is nil)'
104 104 elsif points == problem.full_score
105 105 submission.grader_comment = 'PASSED: ' + comment
106 + elsif result[:comment].chomp =~ /^[\[\]P]+$/
107 + submission.grader_comment = 'PASSED: ' + comment + '(inconsistent score)'
106 108 else
107 109 submission.grader_comment = 'FAILED: ' + comment
108 110 end
109 111 submission.compiler_message = result[:cmp_msg] or ''
110 112 submission.save
111 113 end
112 114
113 115 end
114 116
115 117 end
Show file before | Show file after | Raw diff | Download diff | Ignore whitespace Increase context Show commentsHide comments
@@ -1,10 +1,13
1 + /*
2 + LANG: C
3 + */
1 4 #include <stdio.h>
2 5
3 6 int main()
4 7 {
5 8 int a,
6 9 scanf("%d %d",&a,&b);
7 10 printf("%d\n",a+b);
8 11 return 0;
9 12 }
10 13
Show file before | Show file after | Raw diff | Download diff | Ignore whitespace Increase context Show commentsHide comments
@@ -1,10 +1,13
1 + /*
2 + LANG: C
3 + */
1 4 #include <stdio.h>
2 5
3 6 int main()
4 7 {
5 8 int a,b;
6 9 scanf("%d %d",&a,&b);
7 10 printf("%d\n",a+b);
8 11 return 0;
9 12 }
10 13
Show file before | Show file after | Raw diff | Download diff | Ignore whitespace Increase context Show commentsHide comments
@@ -1,228 +1,228
1 1 require File.join(File.dirname(__FILE__),'spec_helper')
2 2 require File.join(File.dirname(__FILE__),'engine_spec_helper')
3 3
4 4 describe "A grader engine, when grading submissions" do
5 5
6 6 include GraderEngineHelperMethods
7 7
8 8 before(:each) do
9 9 @config = Grader::Configuration.get_instance
10 10
11 11 # this test is from Pong
12 12 @problem_test_normal = stub(Problem,
13 13 :id => 1, :name => 'test_normal',
14 14 :full_score => 135)
15 15 @user_user1 = stub(User,
16 16 :id => 1, :login => 'user1')
17 17
18 18 @engine = Grader::Engine.new
19 19 init_sandbox
20 20 end
21 21
22 22 it "should grade normal submission" do
23 23 grader_should(:grade => "test1_correct.c",
24 24 :on => @problem_test_normal,
25 25 :and_report => {
26 26 :score => 135,
27 27 :comment => /^PASSED/})
28 28 end
29 29
30 30
31 31 it "should produce error message when submission cannot compile" do
32 32 grader_should(:grade => "test1_compile_error.c",
33 33 :on => @problem_test_normal,
34 34 :and_report => {
35 35 :score => 0,
36 - :comment => 'FAILED: compile error',
36 + :comment => 'FAILED: compilation error',
37 37 :compiler_message => /[Ee]rror/})
38 38 end
39 39
40 40 it "should produce timeout error when submission runs forever" do
41 41 @problem_test_timeout = stub(Problem,
42 42 :id => 1, :name => 'test_timeout',
43 43 :full_score => 10)
44 44 grader_should(:grade => "test2_timeout.c",
45 45 :on => @problem_test_timeout,
46 46 :and_report => {
47 47 :score => 0,
48 48 :comment => 'FAILED: TT'})
49 49 end
50 50
51 51 it "should produce timeout error correctly when submission runs slower than expected in less than a second" do
52 52 @problem_test_timeout = stub(Problem,
53 53 :id => 1, :name => 'test_timeout',
54 54 :full_score => 20)
55 55 grader_should(:grade => "test2_1-5sec.c",
56 56 :on => @problem_test_timeout,
57 57 :and_report => {
58 58 :score => 10,
59 59 :comment => 'FAILED: TP'})
60 60 end
61 61
62 62 it "should produce runtime error when submission uses too much static memory" do
63 63 @problem_test_memory = stub(Problem,
64 64 :id => 1, :name => 'test_memory',
65 65 :full_score => 20)
66 66 grader_should(:grade => "add_too_much_memory_static.c",
67 67 :on => @problem_test_memory,
68 68 :and_report => {
69 69 :score => 10,
70 70 :comment => /FAILED: [^P]P/})
71 71 end
72 72
73 73 it "should not allow submission to allocate too much dynamic memory" do
74 74 @problem_test_memory = stub(Problem,
75 75 :id => 1, :name => 'test_memory',
76 76 :full_score => 20)
77 77 grader_should(:grade => "add_too_much_memory_dynamic.c",
78 78 :on => @problem_test_memory,
79 79 :and_report => {
80 80 :score => 10,
81 81 :comment => /FAILED: [^P]P/})
82 82 end
83 83
84 84 it "should score test runs correctly when submission fails in some test case" do
85 85 grader_should(:grade => "add_fail_test_case_1.c",
86 86 :on => @problem_test_normal,
87 87 :and_report => {
88 88 :score => 105,
89 89 :comment => /^FAILED:/})
90 90 end
91 91
92 92 it "should fail submission with non-zero exit status" do
93 93 grader_should(:grade => "add_nonzero_exit_status.c",
94 94 :on => @problem_test_normal,
95 95 :and_report => {
96 96 :score => 0,
97 97 :comment => /^FAILED:/})
98 98 end
99 99
100 100 it "should not allow malicious submission to see PROBLEM_HOME" do
101 101 problem_test_yesno = stub(Problem,
102 102 :id => 1, :name => 'test_yesno',
103 103 :full_score => 10)
104 104 grader_should(:grade => "yesno_access_problem_home.c",
105 105 :on => problem_test_yesno,
106 106 :and_report => {
107 107 :score => 0,
108 108 :comment => /^FAILED:/})
109 109 end
110 110
111 111 it "should not allow malicious submission to open files" do
112 112 problem_test_yesno = stub(Problem,
113 113 :id => 1, :name => 'test_yesno',
114 114 :full_score => 10)
115 115 grader_should(:grade => "yesno_open_file.c",
116 116 :on => problem_test_yesno,
117 117 :and_report => {
118 118 :score => 0,
119 119 :comment => /^FAILED:/})
120 120 end
121 121
122 122 def grader_should(args)
123 123 @user1 = stub(User,
124 124 :id => 1, :login => 'user1')
125 125 submission =
126 126 create_submission_from_file(1, @user1, args[:on], args[:grade])
127 127 submission.should_receive(:graded_at=)
128 128
129 129 expected_score = args[:and_report][:score]
130 130 expected_comment = args[:and_report][:comment]
131 131 if args[:and_report][:compiler_message]!=nil
132 132 expected_compiler_message = args[:and_report][:compiler_message]
133 133 else
134 134 expected_compiler_message = ''
135 135 end
136 136
137 137 submission.should_receive(:points=).with(expected_score)
138 138 submission.should_receive(:grader_comment=).with(expected_comment)
139 139 submission.should_receive(:compiler_message=).with(expected_compiler_message)
140 140 submission.should_receive(:save)
141 141
142 142 @engine.grade(submission)
143 143 end
144 144
145 145 protected
146 146
147 147 def create_normal_submission_mock_from_file(source_fname)
148 148 create_submission_from_file(1, @user_user1, @problem_test_normal, source_fname)
149 149 end
150 150
151 151 end
152 152
153 153 describe "A grader engine, when grading test requests" do
154 154
155 155 include GraderEngineHelperMethods
156 156
157 157 before(:each) do
158 158 @config = Grader::Configuration.get_instance
159 159 @engine = Grader::Engine.new(Grader::TestRequestRoomMaker.new,
160 160 Grader::TestRequestReporter.new)
161 161 init_sandbox
162 162 end
163 163
164 164 it "should report error if there is no problem template" do
165 165 problem = stub(Problem,
166 166 :id => 1, :name => 'nothing')
167 167 grader_should(:grade => 'test1_correct.c',
168 168 :on => problem,
169 169 :with => 'in1.txt',
170 170 :and_report => {
171 171 :graded_at= => nil,
172 172 :compiler_message= => '',
173 173 :grader_comment= => '',
174 174 :running_stat= => /template not found/,
175 175 :save => nil})
176 176 end
177 177
178 178 it "should run test request and produce output file" do
179 179 problem = stub(Problem,
180 180 :id => 1, :name => 'test_normal')
181 181 grader_should(:grade => 'test1_correct.c',
182 182 :on => problem,
183 183 :with => 'in1.txt',
184 184 :and_report => {
185 185 :graded_at= => nil,
186 186 :compiler_message= => '',
187 187 :grader_comment= => '',
188 188 :running_stat= => /0.0 sec./,
189 189 :output_file_name= => lambda { |fname|
190 190 File.exists?(fname).should be_true
191 191 },
192 192 :save => nil})
193 193 end
194 194
195 195 it "should clean up problem directory after running test request" do
196 196 problem = stub(Problem,
197 197 :id => 1, :name => 'test_normal')
198 198 grader_should(:grade => 'test1_correct.c',
199 199 :on => problem,
200 200 :with => 'in1.txt',
201 201 :and_report => {
202 202 :graded_at= => nil,
203 203 :compiler_message= => '',
204 204 :grader_comment= => '',
205 205 :running_stat= => nil,
206 206 :output_file_name= => nil,
207 207 :save => nil})
208 208 File.exists?(@config.user_result_dir + "/test_request/test_normal/test_cases/1/input-1.txt").should be_false
209 209 end
210 210
211 211 it "should compile test request with error and report compilation error" do
212 212 problem = stub(Problem,
213 213 :id => 1, :name => 'test_normal')
214 214 grader_should(:grade => 'test1_compile_error.c',
215 215 :on => problem,
216 216 :with => 'in1.txt',
217 217 :and_report => {
218 218 :graded_at= => nil,
219 219 :compiler_message= => /.+/,
220 220 :grader_comment= => /[Cc]ompil.*error/,
221 221 :running_stat= => '',
222 222 :save => nil})
223 223 end
224 224
225 225 it "should report exit status" do
226 226 problem = stub(Problem,
227 227 :id => 1, :name => 'test_normal')
228 228 grader_should(:grade => 'add_nonzero_exit_status.c',
You need to be logged in to leave comments. Login now