cafe-grader-judge-scripts

Commit r37:effd16a52c43 public

parent | child
Description:
[grader] fixed bug #6 and #9 git-svn-id: http://theory.cpe.ku.ac.th/grader/judge/trunk/scripts@139 6386c4cd-e34a-4fa8-8920-d93eb39b512e
Commit status:
[Not Reviewed]
References:
default
Diff options:
Raw Diff | Patch Diff | Download Diff | Ignore whitespace | Increase context
Comments:
0 Commit comments 0 Inline Comments
Unresolved TODOs:
There are no unresolved TODOs
Side by Side Unified
Expand All Files Collapse All Files Wide Mode Diff
Add another comment
Browse Files

r37:effd16a52c43 - - 5 files changed: 13 inserted, 3 deleted

Show file before | Show file after | Raw diff | Download diff | Ignore whitespace Increase context Show commentsHide comments
@@ -1,82 +1,84
1 1 module Grader
2 2
3 3 # This singleton class holds basic configurations for grader. When
4 4 # running in each mode, grader uses resources from different
5 5 # directories and outputs differently. Usually the attributes name
6 6 # are descriptive; below we explain more on each attributes.
7 7 class Configuration
8 8 # Rails' environment: "development", "production"
9 9 attr_accessor :rails_env
10 10
11 11 # Grader looks for problem [prob] in problem_dir/[prob], and store
12 12 # execution results for submission [x] of user [u] in directory
13 13 # user_result_dir/[u]/[x]
14 14 attr_accessor :problems_dir
15 15 attr_accessor :user_result_dir
16 16
17 17 # If report_grader=true, the grader would add a row in model
18 18 # GraderProcess. It would report itself with grader_hostname and
19 19 # process id.
20 20 attr_accessor :report_grader
21 21 attr_accessor :grader_hostname
22 22
23 23 # If talkative=true, grader would report status to console. If
24 24 # logging=true, grader would report status to a log file located
25 25 # in log_dir, in a file name mode.options.pid. TODO: defined
26 26 # log file naming.
27 27 attr_accessor :talkative
28 28 attr_accessor :logging
29 29 attr_accessor :log_dir
30 30
31 31 # These are directories related to the test interface.
32 32 attr_accessor :test_request_input_base_dir
33 33 attr_accessor :test_request_output_base_dir
34 34 attr_accessor :test_request_problem_templates_dir
35 35
36 36 # Comment received from the grading script will be filtered
37 37 # through Configuration#report_comment. How this method behave
38 38 # depends on this option; right now only two formats, :short and
39 39 # :long
40 40 attr_accessor :comment_report_style
41 41
42 42 def report_comment(comment)
43 43 case comment_report_style
44 44 when :short
45 - if comment.chomp =~ /^P+$/ # all P's
45 + if comment.chomp =~ /^[\[\]P]+$/ # all P's
46 46 'passed'
47 + elsif comment.chomp =~ /[Cc]ompil.*[Ee]rror/
48 + 'compilation error'
47 49 else
48 50 'failed'
49 51 end
50 52
51 53 when :full
52 54 comment.chomp
53 55 end
54 56 end
55 57
56 58 # Codes for singleton
57 59 private_class_method :new
58 60
59 61 @@instance = nil
60 62
61 63 def self.get_instance
62 64 if @@instance==nil
63 65 @@instance = new
64 66 end
65 67 @@instance
66 68 end
67 69
68 70 private
69 71 def initialize
70 72 @talkative = false
71 73 @log_file = nil
72 74 @report_grader = false
73 75 @grader_hostname = `hostname`.chomp
74 76
75 77 @rails_env = 'development'
76 78
77 79 @comment_report_style = :full
78 80 end
79 81
80 82 end
81 83
82 84 end
Show file before | Show file after | Raw diff | Download diff | Ignore whitespace Increase context Show commentsHide comments
@@ -1,115 +1,117
1 1 module Grader
2 2
3 3 class SubmissionRoomMaker
4 4 def initialize
5 5 @config = Grader::Configuration.get_instance
6 6 end
7 7
8 8 def produce_grading_room(submission)
9 9 user = submission.user
10 10 problem = submission.problem
11 11 grading_room = "#{@config.user_result_dir}/" +
12 12 "#{user.login}/#{problem.name}/#{submission.id}"
13 13
14 14 FileUtils.mkdir_p(grading_room)
15 15 grading_room
16 16 end
17 17
18 18 def find_problem_home(submission)
19 19 problem = submission.problem
20 20 "#{@config.problems_dir}/#{problem.name}"
21 21 end
22 22
23 23 def save_source(submission,source_name)
24 24 dir = self.produce_grading_room(submission)
25 25 f = File.open("#{dir}/#{source_name}","w")
26 26 f.write(submission.source)
27 27 f.close
28 28 end
29 29
30 30 def clean_up(submission)
31 31 end
32 32 end
33 33
34 34 class SubmissionReporter
35 35 def initialize
36 36 @config = Grader::Configuration.get_instance
37 37 end
38 38
39 39 def report(sub,test_result_dir)
40 40 save_result(sub,read_result(test_result_dir))
41 41 end
42 42
43 43 def report_error(sub,msg)
44 44 save_result(sub,{:points => 0,
45 45 :comment => "Grading error: #{msg}" })
46 46 end
47 47
48 48 protected
49 49 def read_result(test_result_dir)
50 50 cmp_msg_fname = "#{test_result_dir}/compiler_message"
51 51 if FileTest.exist?(cmp_msg_fname)
52 52 cmp_file = File.open(cmp_msg_fname)
53 53 cmp_msg = cmp_file.read
54 54 cmp_file.close
55 55 else
56 56 cmp_msg = ""
57 57 end
58 58
59 59 result_fname = "#{test_result_dir}/result"
60 60 comment_fname = "#{test_result_dir}/comment"
61 61 if FileTest.exist?(result_fname)
62 62 comment = ""
63 63 begin
64 64 result_file = File.open(result_fname)
65 65 result = result_file.readline.to_i
66 66 result_file.close
67 67 rescue
68 68 result = 0
69 69 comment = "error reading result file."
70 70 end
71 71
72 72 begin
73 73 comment_file = File.open(comment_fname)
74 74 comment += comment_file.readline.chomp
75 75 comment_file.close
76 76 rescue
77 77 comment += ""
78 78 end
79 79
80 80 return {:points => result,
81 81 :comment => comment,
82 82 :cmp_msg => cmp_msg}
83 83 else
84 84 if FileTest.exist?("#{test_result_dir}/a.out")
85 85 return {:points => 0,
86 86 :comment => 'error during grading',
87 87 :cmp_msg => cmp_msg}
88 88 else
89 89 return {:points => 0,
90 - :comment => 'compile error',
90 + :comment => 'compilation error',
91 91 :cmp_msg => cmp_msg}
92 92 end
93 93 end
94 94 end
95 95
96 96 def save_result(submission,result)
97 97 problem = submission.problem
98 98 submission.graded_at = Time.now
99 99 points = result[:points]
100 100 submission.points = points
101 101 comment = @config.report_comment(result[:comment])
102 102 if problem == nil
103 103 submission.grader_comment = 'PASSED: ' + comment + '(problem is nil)'
104 104 elsif points == problem.full_score
105 105 submission.grader_comment = 'PASSED: ' + comment
106 + elsif result[:comment].chomp =~ /^[\[\]P]+$/
107 + submission.grader_comment = 'PASSED: ' + comment + '(inconsistent score)'
106 108 else
107 109 submission.grader_comment = 'FAILED: ' + comment
108 110 end
109 111 submission.compiler_message = result[:cmp_msg] or ''
110 112 submission.save
111 113 end
112 114
113 115 end
114 116
115 117 end
Show file before | Show file after | Raw diff | Download diff | Ignore whitespace Increase context Show commentsHide comments
@@ -1,10 +1,13
1 + /*
2 + LANG: C
3 + */
1 4 #include <stdio.h>
2 5
3 6 int main()
4 7 {
5 8 int a,
6 9 scanf("%d %d",&a,&b);
7 10 printf("%d\n",a+b);
8 11 return 0;
9 12 }
10 13
Show file before | Show file after | Raw diff | Download diff | Ignore whitespace Increase context Show commentsHide comments
@@ -1,10 +1,13
1 + /*
2 + LANG: C
3 + */
1 4 #include <stdio.h>
2 5
3 6 int main()
4 7 {
5 8 int a,b;
6 9 scanf("%d %d",&a,&b);
7 10 printf("%d\n",a+b);
8 11 return 0;
9 12 }
10 13
Show file before | Show file after | Raw diff | Download diff | Ignore whitespace Increase context Show commentsHide comments
@@ -1,132 +1,132
1 1 require File.join(File.dirname(__FILE__),'spec_helper')
2 2 require File.join(File.dirname(__FILE__),'engine_spec_helper')
3 3
4 4 describe "A grader engine, when grading submissions" do
5 5
6 6 include GraderEngineHelperMethods
7 7
8 8 before(:each) do
9 9 @config = Grader::Configuration.get_instance
10 10
11 11 # this test is from Pong
12 12 @problem_test_normal = stub(Problem,
13 13 :id => 1, :name => 'test_normal',
14 14 :full_score => 135)
15 15 @user_user1 = stub(User,
16 16 :id => 1, :login => 'user1')
17 17
18 18 @engine = Grader::Engine.new
19 19 init_sandbox
20 20 end
21 21
22 22 it "should grade normal submission" do
23 23 grader_should(:grade => "test1_correct.c",
24 24 :on => @problem_test_normal,
25 25 :and_report => {
26 26 :score => 135,
27 27 :comment => /^PASSED/})
28 28 end
29 29
30 30
31 31 it "should produce error message when submission cannot compile" do
32 32 grader_should(:grade => "test1_compile_error.c",
33 33 :on => @problem_test_normal,
34 34 :and_report => {
35 35 :score => 0,
36 - :comment => 'FAILED: compile error',
36 + :comment => 'FAILED: compilation error',
37 37 :compiler_message => /[Ee]rror/})
38 38 end
39 39
40 40 it "should produce timeout error when submission runs forever" do
41 41 @problem_test_timeout = stub(Problem,
42 42 :id => 1, :name => 'test_timeout',
43 43 :full_score => 10)
44 44 grader_should(:grade => "test2_timeout.c",
45 45 :on => @problem_test_timeout,
46 46 :and_report => {
47 47 :score => 0,
48 48 :comment => 'FAILED: TT'})
49 49 end
50 50
51 51 it "should produce timeout error correctly when submission runs slower than expected in less than a second" do
52 52 @problem_test_timeout = stub(Problem,
53 53 :id => 1, :name => 'test_timeout',
54 54 :full_score => 20)
55 55 grader_should(:grade => "test2_1-5sec.c",
56 56 :on => @problem_test_timeout,
57 57 :and_report => {
58 58 :score => 10,
59 59 :comment => 'FAILED: TP'})
60 60 end
61 61
62 62 it "should produce runtime error when submission uses too much static memory" do
63 63 @problem_test_memory = stub(Problem,
64 64 :id => 1, :name => 'test_memory',
65 65 :full_score => 20)
66 66 grader_should(:grade => "add_too_much_memory_static.c",
67 67 :on => @problem_test_memory,
68 68 :and_report => {
69 69 :score => 10,
70 70 :comment => /FAILED: [^P]P/})
71 71 end
72 72
73 73 it "should not allow submission to allocate too much dynamic memory" do
74 74 @problem_test_memory = stub(Problem,
75 75 :id => 1, :name => 'test_memory',
76 76 :full_score => 20)
77 77 grader_should(:grade => "add_too_much_memory_dynamic.c",
78 78 :on => @problem_test_memory,
79 79 :and_report => {
80 80 :score => 10,
81 81 :comment => /FAILED: [^P]P/})
82 82 end
83 83
84 84 it "should score test runs correctly when submission fails in some test case" do
85 85 grader_should(:grade => "add_fail_test_case_1.c",
86 86 :on => @problem_test_normal,
87 87 :and_report => {
88 88 :score => 105,
89 89 :comment => /^FAILED:/})
90 90 end
91 91
92 92 it "should fail submission with non-zero exit status" do
93 93 grader_should(:grade => "add_nonzero_exit_status.c",
94 94 :on => @problem_test_normal,
95 95 :and_report => {
96 96 :score => 0,
97 97 :comment => /^FAILED:/})
98 98 end
99 99
100 100 it "should not allow malicious submission to see PROBLEM_HOME" do
101 101 problem_test_yesno = stub(Problem,
102 102 :id => 1, :name => 'test_yesno',
103 103 :full_score => 10)
104 104 grader_should(:grade => "yesno_access_problem_home.c",
105 105 :on => problem_test_yesno,
106 106 :and_report => {
107 107 :score => 0,
108 108 :comment => /^FAILED:/})
109 109 end
110 110
111 111 it "should not allow malicious submission to open files" do
112 112 problem_test_yesno = stub(Problem,
113 113 :id => 1, :name => 'test_yesno',
114 114 :full_score => 10)
115 115 grader_should(:grade => "yesno_open_file.c",
116 116 :on => problem_test_yesno,
117 117 :and_report => {
118 118 :score => 0,
119 119 :comment => /^FAILED:/})
120 120 end
121 121
122 122 def grader_should(args)
123 123 @user1 = stub(User,
124 124 :id => 1, :login => 'user1')
125 125 submission =
126 126 create_submission_from_file(1, @user1, args[:on], args[:grade])
127 127 submission.should_receive(:graded_at=)
128 128
129 129 expected_score = args[:and_report][:score]
130 130 expected_comment = args[:and_report][:comment]
131 131 if args[:and_report][:compiler_message]!=nil
132 132 expected_compiler_message = args[:and_report][:compiler_message]
You need to be logged in to leave comments. Login now