cafe-grader-judge-scripts

Commit r160:f0502be39e3b public

parent | child
Description:
add run_stat checking
Commit status:
[Not Reviewed]
References:
algo
Diff options:
Raw Diff | Patch Diff | Download Diff | Show whitespace | Increase context
Comments:
0 Commit comments 0 Inline Comments
Unresolved TODOs:
There are no unresolved TODOs
Side by Side Unified
Expand All Files Collapse All Files Wide Mode Diff
Add another comment
Browse Files

r160:f0502be39e3b - - 3 files changed: 48 inserted, 10 deleted

Show file before | Show file after | Raw diff | Download diff | Show whitespace Increase context Show commentsHide comments
@@ -57,66 +57,84
57 def read_result(test_result_dir)
57 def read_result(test_result_dir)
58 cmp_msg_fname = "#{test_result_dir}/compiler_message"
58 cmp_msg_fname = "#{test_result_dir}/compiler_message"
59 if FileTest.exist?(cmp_msg_fname)
59 if FileTest.exist?(cmp_msg_fname)
60 cmp_file = File.open(cmp_msg_fname)
60 cmp_file = File.open(cmp_msg_fname)
61 cmp_msg = cmp_file.read
61 cmp_msg = cmp_file.read
62 cmp_file.close
62 cmp_file.close
63 else
63 else
64 cmp_msg = ""
64 cmp_msg = ""
65 end
65 end
66
66
67 result_fname = "#{test_result_dir}/result"
67 result_fname = "#{test_result_dir}/result"
68 comment_fname = "#{test_result_dir}/comment"
68 comment_fname = "#{test_result_dir}/comment"
69 + runstat_fname = "#{test_result_dir}/run_stat"
69 if FileTest.exist?(result_fname)
70 if FileTest.exist?(result_fname)
70 comment = ""
71 comment = ""
71 begin
72 begin
72 result_file = File.open(result_fname)
73 result_file = File.open(result_fname)
73 result = result_file.readline.to_i
74 result = result_file.readline.to_i
74 result_file.close
75 result_file.close
75 rescue
76 rescue
76 result = 0
77 result = 0
77 comment = "error reading result file."
78 comment = "error reading result file."
78 end
79 end
79
80
80 begin
81 begin
81 comment_file = File.open(comment_fname)
82 comment_file = File.open(comment_fname)
82 comment += comment_file.readline.chomp
83 comment += comment_file.readline.chomp
83 comment_file.close
84 comment_file.close
84 rescue
85 rescue
85 comment += ""
86 comment += ""
86 end
87 end
87
88
88 - return {:points => result,
89 + begin
89 - :comment => comment,
90 + runstat_file = File.open(runstat_fname)
90 - :cmp_msg => cmp_msg}
91 + max_runtime = runstat_file.readline.to_f
92 + peak_memory = runstat_file.readline.to_i
93 + rescue
94 + max_runtime = -1
95 + peak_memory = -1
96 + end
97 +
98 +
99 + return {points: result,
100 + comment: comment,
101 + cmp_msg: cmp_msg,
102 + max_runtime: max_runtime,
103 + peak_memory: peak_memory
104 + }
91 else
105 else
92 if FileTest.exist?("#{test_result_dir}/a.out")
106 if FileTest.exist?("#{test_result_dir}/a.out")
93 return {:points => 0,
107 return {:points => 0,
94 :comment => 'error during grading',
108 :comment => 'error during grading',
95 :cmp_msg => cmp_msg}
109 :cmp_msg => cmp_msg}
96 else
110 else
97 return {:points => 0,
111 return {:points => 0,
98 :comment => 'compilation error',
112 :comment => 'compilation error',
99 :cmp_msg => cmp_msg}
113 :cmp_msg => cmp_msg}
100 end
114 end
101 end
115 end
102 end
116 end
103
117
104 def save_result(submission,result)
118 def save_result(submission,result)
105 problem = submission.problem
119 problem = submission.problem
106 submission.graded_at = Time.now.gmtime
120 submission.graded_at = Time.now.gmtime
107 points = result[:points]
121 points = result[:points]
108 submission.points = points
122 submission.points = points
109 comment = @config.report_comment(result[:comment])
123 comment = @config.report_comment(result[:comment])
110
124
125 + submission.peak_memory = result[:peak_memory]
126 + submission.max_runtime = result[:max_runtime]
127 + submission.effective_code_length =submission.source.length
128 +
111 #
129 #
112 # TODO: FIX THIS MESSAGE
130 # TODO: FIX THIS MESSAGE
113 #
131 #
114 if problem == nil
132 if problem == nil
115 submission.grader_comment = 'PASSED: ' + comment + '(problem is nil)'
133 submission.grader_comment = 'PASSED: ' + comment + '(problem is nil)'
116 elsif points == problem.full_score
134 elsif points == problem.full_score
117 #submission.grader_comment = 'PASSED: ' + comment
135 #submission.grader_comment = 'PASSED: ' + comment
118 submission.grader_comment = comment
136 submission.grader_comment = comment
119 elsif result[:comment].chomp =~ /^[\[\]P]+$/
137 elsif result[:comment].chomp =~ /^[\[\]P]+$/
120 submission.grader_comment = 'PASSED: ' + comment + '(inconsistent score)'
138 submission.grader_comment = 'PASSED: ' + comment + '(inconsistent score)'
121 else
139 else
122 #submission.grader_comment = 'FAILED: ' + comment
140 #submission.grader_comment = 'FAILED: ' + comment
Show file before | Show file after | Raw diff | Download diff | Show whitespace Increase context Show commentsHide comments
@@ -1447,29 +1447,24
1447 else
1447 else
1448 {
1448 {
1449 // Internal error happened inside the child process and it has been already reported.
1449 // Internal error happened inside the child process and it has been already reported.
1450 box_exit(2);
1450 box_exit(2);
1451 }
1451 }
1452 }
1452 }
1453 if (timeout && total_ms > timeout)
1453 if (timeout && total_ms > timeout)
1454 err("TO: Time limit exceeded");
1454 err("TO: Time limit exceeded");
1455 if (wall_timeout && wall_ms > wall_timeout)
1455 if (wall_timeout && wall_ms > wall_timeout)
1456 err("TO: Time limit exceeded (wall clock)");
1456 err("TO: Time limit exceeded (wall clock)");
1457 flush_line();
1457 flush_line();
1458 fprintf(stderr,"OK\n");
1458 fprintf(stderr,"OK\n");
1459 - print_running_stat(
1460 - (double)wall_ms/1000,
1461 - (double)total_ms/1000,
1462 - (double)sys_ms/1000,
1463 - (mem_peak_kb + 1023) / 1024);
1464 box_exit(0);
1459 box_exit(0);
1465 }
1460 }
1466 if (WIFSIGNALED(stat))
1461 if (WIFSIGNALED(stat))
1467 {
1462 {
1468 box_pid = 0;
1463 box_pid = 0;
1469 meta_printf("exitsig:%d\n", WTERMSIG(stat));
1464 meta_printf("exitsig:%d\n", WTERMSIG(stat));
1470 final_stats(&rus);
1465 final_stats(&rus);
1471 err("SG: Caught fatal signal %d%s", WTERMSIG(stat), (syscall_count ? "" : " during startup"));
1466 err("SG: Caught fatal signal %d%s", WTERMSIG(stat), (syscall_count ? "" : " during startup"));
1472 }
1467 }
1473 if (WIFSTOPPED(stat))
1468 if (WIFSTOPPED(stat))
1474 {
1469 {
1475 int sig = WSTOPSIG(stat);
1470 int sig = WSTOPSIG(stat);
Show file before | Show file after | Raw diff | Download diff | Show whitespace Increase context Show commentsHide comments
@@ -22,55 +22,73
22 INCORRECT_MARK
22 INCORRECT_MARK
23 elsif comment =~ /[Cc]orrect/
23 elsif comment =~ /[Cc]orrect/
24 CORRECT_MARK
24 CORRECT_MARK
25 elsif comment =~ /[Tt]ime/
25 elsif comment =~ /[Tt]ime/
26 TIMEOUT_MARK
26 TIMEOUT_MARK
27 elsif res = /^[Cc]omment:(.*)$/.match(comment)
27 elsif res = /^[Cc]omment:(.*)$/.match(comment)
28 res[1]
28 res[1]
29 else
29 else
30 RUN_ERROR_MARK # these are run time errors
30 RUN_ERROR_MARK # these are run time errors
31 end
31 end
32 end
32 end
33
33
34 + def extract_time(t)
35 + puts "TIME: #{t}"
36 + if (result=/^(.*)r(.*)u(.*)s(.*)m/.match(t))
37 + {:real => result[1], :user => result[2], :sys => result[3], :mem => result[4]}
38 + else
39 + #{:real => 0, :user => 0, :sys => 0}
40 + #puts "ERROR READING RUNNING TIME: #{t}"
41 + raise "Error reading running time: #{t}"
42 + end
43 + end
44 +
34 problem_home = ENV['PROBLEM_HOME']
45 problem_home = ENV['PROBLEM_HOME']
35 require "#{problem_home}/script/test_dsl.rb"
46 require "#{problem_home}/script/test_dsl.rb"
36 load "#{problem_home}/test_cases/all_tests.cfg"
47 load "#{problem_home}/test_cases/all_tests.cfg"
37 problem = Problem.get_instance
48 problem = Problem.get_instance
38
49
39 if problem.well_formed? == false
50 if problem.well_formed? == false
40 log "The problem specification is not well formed."
51 log "The problem specification is not well formed."
41 exit(127)
52 exit(127)
42 end
53 end
43
54
44 all_score = 0
55 all_score = 0
45 all_comment = ''
56 all_comment = ''
57 + peak_memory = -1
58 + max_runtime = -1
46 (1..(problem.runs.length-1)).each do |k|
59 (1..(problem.runs.length-1)).each do |k|
47 log "grade run #{k}"
60 log "grade run #{k}"
48 run = problem.runs[k]
61 run = problem.runs[k]
49 run_score = nil
62 run_score = nil
50 run_comment = ''
63 run_comment = ''
51 run_comment_short = ''
64 run_comment_short = ''
52 run.tests.each do |test_num|
65 run.tests.each do |test_num|
53 result_file_name = "#{test_num}/result"
66 result_file_name = "#{test_num}/result"
54 if not File.exists?(result_file_name)
67 if not File.exists?(result_file_name)
55 run_comment += "result file for test #{test_num} not found\n"
68 run_comment += "result file for test #{test_num} not found\n"
56 run_comment_short += RUN_ERROR_MARK
69 run_comment_short += RUN_ERROR_MARK
57 log "Cannot find the file #{test_num}/result!"
70 log "Cannot find the file #{test_num}/result!"
58 else
71 else
59 result_file = File.new(result_file_name, "r")
72 result_file = File.new(result_file_name, "r")
60 result_file_lines = result_file.readlines
73 result_file_lines = result_file.readlines
61 - if result_file_lines.length>=2
74 + if result_file_lines.length>=3
62 current_run_score = result_file_lines[1].to_i
75 current_run_score = result_file_lines[1].to_i
63 run_comment += result_file_lines[0]
76 run_comment += result_file_lines[0]
64 run_comment_short += char_comment(result_file_lines[0].chomp)
77 run_comment_short += char_comment(result_file_lines[0].chomp)
78 +
79 + #update max runtime & memory
80 + run_stat = extract_time result_file_lines[2]
81 + peak_memory = [peak_memory,run_stat[:mem].to_i].max
82 + max_runtime = [max_runtime,run_stat[:user].to_f + run_stat[:sys].to_f].max
65 else
83 else
66 current_run_score = 0
84 current_run_score = 0
67 run_comment += "result file for test #{test_num} error\n"
85 run_comment += "result file for test #{test_num} error\n"
68 run_comment_short += RUN_ERROR_MARK
86 run_comment_short += RUN_ERROR_MARK
69 log "Error in #{test_num}/result!"
87 log "Error in #{test_num}/result!"
70 end
88 end
71
89
72 # the score of this run should be the minimum of the score for
90 # the score of this run should be the minimum of the score for
73 # each test case
91 # each test case
74 if (run_score==nil) or (run_score>current_run_score)
92 if (run_score==nil) or (run_score>current_run_score)
75 run_score = current_run_score
93 run_score = current_run_score
76 end
94 end
@@ -96,13 +114,20
96 all_comment += run_comment_short
114 all_comment += run_comment_short
97 end
115 end
98
116
99 result_file = File.new("result", "w")
117 result_file = File.new("result", "w")
100 result_file.write all_score
118 result_file.write all_score
101 result_file.write "\n"
119 result_file.write "\n"
102 result_file.close
120 result_file.close
103
121
104 comment_file = File.new("comment", "w")
122 comment_file = File.new("comment", "w")
105 comment_file.write "#{all_comment}\n"
123 comment_file.write "#{all_comment}\n"
106 comment_file.close
124 comment_file.close
107
125
108 - log "score = #{all_score} comment = #{all_comment}"
126 +
127 + File.open("run_stat","w") do |file|
128 + file.puts max_runtime
129 + file.puts peak_memory
130 + end
131 +
132 + log "score = #{all_score}\ncomment = #{all_comment}"
133 + log "max_runtime = #{max_runtime}\npeak_memory = #{peak_memory}"
You need to be logged in to leave comments. Login now