cafe-grader-judge-scripts

Commit r32:f9158ac7d119 public

parent | child
Description:
fix test run scoring bug git-svn-id: http://theory.cpe.ku.ac.th/grader/judge/trunk/scripts@126 6386c4cd-e34a-4fa8-8920-d93eb39b512e
Commit status:
[Not Reviewed]
References:
default
Diff options:
Raw Diff | Patch Diff | Download Diff | Show whitespace | Increase context
Comments:
0 Commit comments 0 Inline Comments
Unresolved TODOs:
There are no unresolved TODOs
Side by Side Unified
Expand All Files Collapse All Files Wide Mode Diff
Add another comment
Browse Files

r32:f9158ac7d119 - - 3 files changed: 35 inserted, 0 deleted

Show file before | Show file after | Raw diff | Download diff | Show whitespace Increase context Show commentsHide comments
@@ -0,0 +1,13
1 + #include <stdio.h>
2 +
3 + int main()
4 + {
5 + int a,b;
6 + scanf("%d %d",&a,&b);
7 + if((a==1) && (b==1))
8 + printf("100\n");
9 + else
10 + printf("%d\n",a+b);
11 + return 0;
12 + }
13 +
Show file before | Show file after | Raw diff | Download diff | Show whitespace Increase context Show commentsHide comments
@@ -42,45 +42,59
42 all_score = 0
42 all_score = 0
43 all_comment = ''
43 all_comment = ''
44 (1..(problem.runs.length-1)).each do |k|
44 (1..(problem.runs.length-1)).each do |k|
45 log "grade run #{k}"
45 log "grade run #{k}"
46 run = problem.runs[k]
46 run = problem.runs[k]
47 run_score = 0
47 run_score = 0
48 run_comment = ''
48 run_comment = ''
49 run_comment_short = ''
49 run_comment_short = ''
50 run.tests.each do |test_num|
50 run.tests.each do |test_num|
51 result_file_name = "#{test_num}/result"
51 result_file_name = "#{test_num}/result"
52 if not File.exists?(result_file_name)
52 if not File.exists?(result_file_name)
53 run_comment += "result file for test #{test_num} not found\n"
53 run_comment += "result file for test #{test_num} not found\n"
54 run_comment_short += RUN_ERROR_MARK
54 run_comment_short += RUN_ERROR_MARK
55 log "Cannot find the file #{test_num}/result!"
55 log "Cannot find the file #{test_num}/result!"
56 else
56 else
57 result_file = File.new(result_file_name, "r")
57 result_file = File.new(result_file_name, "r")
58 result_file_lines = result_file.readlines
58 result_file_lines = result_file.readlines
59 run_score = run_score + result_file_lines[1].to_i
59 run_score = run_score + result_file_lines[1].to_i
60 run_comment += result_file_lines[0]
60 run_comment += result_file_lines[0]
61 run_comment_short += char_comment(result_file_lines[0])
61 run_comment_short += char_comment(result_file_lines[0])
62 result_file.close
62 result_file.close
63 end
63 end
64 end
64 end
65
65
66 + # find total score for this run
67 + run_total_score = 0
68 + problem = Problem.get_instance
69 + run.tests.each { |test_num| run_total_score += problem.get_score(test_num) }
70 +
71 + if run_total_score!=run_score # fail in some test cases, fail the run
72 + run_score = 0
73 + end
74 +
66 run_result_file = File.new("result-#{k}", "w")
75 run_result_file = File.new("result-#{k}", "w")
67 run_result_file.write run_score
76 run_result_file.write run_score
68 run_result_file.write "\n"
77 run_result_file.write "\n"
69 run_result_file.close
78 run_result_file.close
70
79
71 run_comment_file = File.new("comment-#{k}", "w")
80 run_comment_file = File.new("comment-#{k}", "w")
72 run_comment_file.write "#{run_comment}\n"
81 run_comment_file.write "#{run_comment}\n"
73 run_comment_file.close
82 run_comment_file.close
74
83
75 all_score = all_score + run_score
84 all_score = all_score + run_score
85 +
86 + # append comment for test run with many test cases
87 + if run.tests.length > 1
88 + run_comment_short = '[' + run_comment_short + ']'
89 + end
76 all_comment += run_comment_short
90 all_comment += run_comment_short
77 end
91 end
78
92
79 result_file = File.new("result", "w")
93 result_file = File.new("result", "w")
80 result_file.write all_score
94 result_file.write all_score
81 result_file.write "\n"
95 result_file.write "\n"
82 result_file.close
96 result_file.close
83
97
84 comment_file = File.new("comment", "w")
98 comment_file = File.new("comment", "w")
85 comment_file.write "#{all_comment}\n"
99 comment_file.write "#{all_comment}\n"
86 comment_file.close
100 comment_file.close
Show file before | Show file after | Raw diff | Download diff | Show whitespace Increase context Show commentsHide comments
@@ -60,48 +60,56
60 end
60 end
61
61
62 it "should produce runtime error when submission uses too much static memory" do
62 it "should produce runtime error when submission uses too much static memory" do
63 @problem_test_memory = stub(Problem,
63 @problem_test_memory = stub(Problem,
64 :id => 1, :name => 'test_memory',
64 :id => 1, :name => 'test_memory',
65 :full_score => 20)
65 :full_score => 20)
66 grader_should(:grade => "add_too_much_memory_static.c",
66 grader_should(:grade => "add_too_much_memory_static.c",
67 :on => @problem_test_memory,
67 :on => @problem_test_memory,
68 :and_report => {
68 :and_report => {
69 :score => 10,
69 :score => 10,
70 :comment => /FAILED: [^P]P/})
70 :comment => /FAILED: [^P]P/})
71 end
71 end
72
72
73 it "should not allow submission to allocate too much dynamic memory" do
73 it "should not allow submission to allocate too much dynamic memory" do
74 @problem_test_memory = stub(Problem,
74 @problem_test_memory = stub(Problem,
75 :id => 1, :name => 'test_memory',
75 :id => 1, :name => 'test_memory',
76 :full_score => 20)
76 :full_score => 20)
77 grader_should(:grade => "add_too_much_memory_dynamic.c",
77 grader_should(:grade => "add_too_much_memory_dynamic.c",
78 :on => @problem_test_memory,
78 :on => @problem_test_memory,
79 :and_report => {
79 :and_report => {
80 :score => 10,
80 :score => 10,
81 :comment => /FAILED: [^P]P/})
81 :comment => /FAILED: [^P]P/})
82 end
82 end
83
83
84 + it "should score test runs correctly when submission fails in some test case" do
85 + grader_should(:grade => "add_fail_test_case_1.c",
86 + :on => @problem_test_normal,
87 + :and_report => {
88 + :score => 105,
89 + :comment => /^FAILED:/})
90 + end
91 +
84 it "should fail submission with non-zero exit status" do
92 it "should fail submission with non-zero exit status" do
85 violated("has not been implemented")
93 violated("has not been implemented")
86 end
94 end
87
95
88 def grader_should(args)
96 def grader_should(args)
89 @user1 = stub(User,
97 @user1 = stub(User,
90 :id => 1, :login => 'user1')
98 :id => 1, :login => 'user1')
91 submission =
99 submission =
92 create_submission_from_file(1, @user1, args[:on], args[:grade])
100 create_submission_from_file(1, @user1, args[:on], args[:grade])
93 submission.should_receive(:graded_at=)
101 submission.should_receive(:graded_at=)
94
102
95 expected_score = args[:and_report][:score]
103 expected_score = args[:and_report][:score]
96 expected_comment = args[:and_report][:comment]
104 expected_comment = args[:and_report][:comment]
97 if args[:and_report][:compiler_message]!=nil
105 if args[:and_report][:compiler_message]!=nil
98 expected_compiler_message = args[:and_report][:compiler_message]
106 expected_compiler_message = args[:and_report][:compiler_message]
99 else
107 else
100 expected_compiler_message = ''
108 expected_compiler_message = ''
101 end
109 end
102
110
103 submission.should_receive(:points=).with(expected_score)
111 submission.should_receive(:points=).with(expected_score)
104 submission.should_receive(:grader_comment=).with(expected_comment)
112 submission.should_receive(:grader_comment=).with(expected_comment)
105 submission.should_receive(:compiler_message=).with(expected_compiler_message)
113 submission.should_receive(:compiler_message=).with(expected_compiler_message)
106 submission.should_receive(:save)
114 submission.should_receive(:save)
107
115
You need to be logged in to leave comments. Login now