cafe-grader-judge-scripts

Commit r160:f0502be39e3b public

parent | child
Description:
add run_stat checking
Commit status:
[Not Reviewed]
References:
algo
Diff options:
Raw Diff | Patch Diff | Download Diff | Ignore whitespace | Increase context
Comments:
0 Commit comments 0 Inline Comments
Unresolved TODOs:
There are no unresolved TODOs
Side by Side Unified
Expand All Files Collapse All Files Wide Mode Diff
Add another comment
Browse Files

r160:f0502be39e3b - - 3 files changed: 49 inserted, 11 deleted

Show file before | Show file after | Raw diff | Download diff | Ignore whitespace Increase context Show commentsHide comments
@@ -20,115 +20,133
20 20 "#{@config.problems_dir}/#{problem.name}"
21 21 end
22 22
23 23 def save_source(submission,source_name)
24 24 dir = self.produce_grading_room(submission)
25 25 f = File.open("#{dir}/#{source_name}","w")
26 26 f.write(submission.source)
27 27 f.close
28 28 end
29 29
30 30 def clean_up(submission)
31 31 end
32 32 end
33 33
34 34 class SubmissionReporter
35 35 def initialize(options={})
36 36 options = {:dry_run => false, :result_collector => nil}.merge(options)
37 37 @config = Grader::Configuration.get_instance
38 38 @dry_run = options[:dry_run]
39 39 @result_collector = options[:result_collector]
40 40 end
41 41
42 42 def report(sub,test_result_dir)
43 43 result = read_result(test_result_dir)
44 44 if @result_collector
45 45 @result_collector.save(sub,
46 46 result)
47 47 end
48 48 save_result(sub,result)
49 49 end
50 50
51 51 def report_error(sub,msg)
52 52 save_result(sub,{:points => 0,
53 53 :comment => "Grading error: #{msg}" })
54 54 end
55 55
56 56 protected
57 57 def read_result(test_result_dir)
58 58 cmp_msg_fname = "#{test_result_dir}/compiler_message"
59 59 if FileTest.exist?(cmp_msg_fname)
60 60 cmp_file = File.open(cmp_msg_fname)
61 61 cmp_msg = cmp_file.read
62 62 cmp_file.close
63 63 else
64 64 cmp_msg = ""
65 65 end
66 66
67 67 result_fname = "#{test_result_dir}/result"
68 - comment_fname = "#{test_result_dir}/comment"
68 + comment_fname = "#{test_result_dir}/comment"
69 + runstat_fname = "#{test_result_dir}/run_stat"
69 70 if FileTest.exist?(result_fname)
70 71 comment = ""
71 72 begin
72 73 result_file = File.open(result_fname)
73 74 result = result_file.readline.to_i
74 75 result_file.close
75 76 rescue
76 77 result = 0
77 78 comment = "error reading result file."
78 79 end
79 80
80 81 begin
81 82 comment_file = File.open(comment_fname)
82 83 comment += comment_file.readline.chomp
83 84 comment_file.close
84 85 rescue
85 86 comment += ""
86 87 end
87 88
88 - return {:points => result,
89 - :comment => comment,
90 - :cmp_msg => cmp_msg}
89 + begin
90 + runstat_file = File.open(runstat_fname)
91 + max_runtime = runstat_file.readline.to_f
92 + peak_memory = runstat_file.readline.to_i
93 + rescue
94 + max_runtime = -1
95 + peak_memory = -1
96 + end
97 +
98 +
99 + return {points: result,
100 + comment: comment,
101 + cmp_msg: cmp_msg,
102 + max_runtime: max_runtime,
103 + peak_memory: peak_memory
104 + }
91 105 else
92 106 if FileTest.exist?("#{test_result_dir}/a.out")
93 107 return {:points => 0,
94 108 :comment => 'error during grading',
95 109 :cmp_msg => cmp_msg}
96 110 else
97 111 return {:points => 0,
98 112 :comment => 'compilation error',
99 113 :cmp_msg => cmp_msg}
100 114 end
101 115 end
102 116 end
103 117
104 118 def save_result(submission,result)
105 119 problem = submission.problem
106 120 submission.graded_at = Time.now.gmtime
107 121 points = result[:points]
108 122 submission.points = points
109 123 comment = @config.report_comment(result[:comment])
110 124
125 + submission.peak_memory = result[:peak_memory]
126 + submission.max_runtime = result[:max_runtime]
127 + submission.effective_code_length =submission.source.length
128 +
111 129 #
112 130 # TODO: FIX THIS MESSAGE
113 131 #
114 132 if problem == nil
115 133 submission.grader_comment = 'PASSED: ' + comment + '(problem is nil)'
116 134 elsif points == problem.full_score
117 135 #submission.grader_comment = 'PASSED: ' + comment
118 136 submission.grader_comment = comment
119 137 elsif result[:comment].chomp =~ /^[\[\]P]+$/
120 138 submission.grader_comment = 'PASSED: ' + comment + '(inconsistent score)'
121 139 else
122 140 #submission.grader_comment = 'FAILED: ' + comment
123 141 submission.grader_comment = comment
124 142 end
125 143 submission.compiler_message = result[:cmp_msg] or ''
126 144
127 145 if not @dry_run
128 146 submission.save
129 147 end
130 148 end
131 149
132 150 end
133 151
134 152 end
Show file before | Show file after | Raw diff | Download diff | Ignore whitespace Increase context Show commentsHide comments
@@ -1411,101 +1411,96
1411 1411 {
1412 1412 sa.sa_handler = signal_alarm;
1413 1413 sigaction(SIGALRM, &sa, NULL);
1414 1414 alarm(1);
1415 1415 }
1416 1416
1417 1417 for(;;)
1418 1418 {
1419 1419 struct rusage rus;
1420 1420 int stat;
1421 1421 pid_t p;
1422 1422 if (timer_tick)
1423 1423 {
1424 1424 check_timeout();
1425 1425 timer_tick = 0;
1426 1426 }
1427 1427 p = wait4(box_pid, &stat, WUNTRACED, &rus);
1428 1428 if (p < 0)
1429 1429 {
1430 1430 if (errno == EINTR)
1431 1431 continue;
1432 1432 die("wait4: %m");
1433 1433 }
1434 1434 if (p != box_pid)
1435 1435 die("wait4: unknown pid %d exited!", p);
1436 1436 if (WIFEXITED(stat))
1437 1437 {
1438 1438 box_pid = 0;
1439 1439 final_stats(&rus);
1440 1440 if (WEXITSTATUS(stat))
1441 1441 {
1442 1442 if (syscall_count)
1443 1443 {
1444 1444 meta_printf("exitcode:%d\n", WEXITSTATUS(stat));
1445 1445 err("RE: Exited with error status %d", WEXITSTATUS(stat));
1446 1446 }
1447 1447 else
1448 1448 {
1449 1449 // Internal error happened inside the child process and it has been already reported.
1450 1450 box_exit(2);
1451 1451 }
1452 1452 }
1453 1453 if (timeout && total_ms > timeout)
1454 1454 err("TO: Time limit exceeded");
1455 1455 if (wall_timeout && wall_ms > wall_timeout)
1456 1456 err("TO: Time limit exceeded (wall clock)");
1457 1457 flush_line();
1458 1458 fprintf(stderr,"OK\n");
1459 - print_running_stat(
1460 - (double)wall_ms/1000,
1461 - (double)total_ms/1000,
1462 - (double)sys_ms/1000,
1463 - (mem_peak_kb + 1023) / 1024);
1464 1459 box_exit(0);
1465 1460 }
1466 1461 if (WIFSIGNALED(stat))
1467 1462 {
1468 1463 box_pid = 0;
1469 1464 meta_printf("exitsig:%d\n", WTERMSIG(stat));
1470 1465 final_stats(&rus);
1471 1466 err("SG: Caught fatal signal %d%s", WTERMSIG(stat), (syscall_count ? "" : " during startup"));
1472 1467 }
1473 1468 if (WIFSTOPPED(stat))
1474 1469 {
1475 1470 int sig = WSTOPSIG(stat);
1476 1471 if (sig == SIGTRAP)
1477 1472 {
1478 1473 if (verbose > 2)
1479 1474 msg("[ptrace status %08x] ", stat);
1480 1475 static int stop_count;
1481 1476 if (!stop_count++) /* Traceme request */
1482 1477 msg(">> Traceme request caught\n");
1483 1478 else
1484 1479 err("SG: Breakpoint");
1485 1480 ptrace(PTRACE_SYSCALL, box_pid, 0, 0);
1486 1481 }
1487 1482 else if (sig == (SIGTRAP | 0x80))
1488 1483 {
1489 1484 if (verbose > 2)
1490 1485 msg("[ptrace status %08x] ", stat);
1491 1486 struct syscall_args a;
1492 1487 static unsigned int sys_tick, last_act;
1493 1488 static arg_t last_sys;
1494 1489 if (++sys_tick & 1) /* Syscall entry */
1495 1490 {
1496 1491 char namebuf[32];
1497 1492 int act;
1498 1493
1499 1494 get_syscall_args(&a, 0);
1500 1495 arg_t sys = a.sys;
1501 1496 msg(">> Syscall %-12s (%08jx,%08jx,%08jx) ", syscall_name(sys, namebuf), (intmax_t) a.arg1, (intmax_t) a.arg2, (intmax_t) a.arg3);
1502 1497 if (!exec_seen)
1503 1498 {
1504 1499 msg("[master] ");
1505 1500 if (sys == NATIVE_NR_execve)
1506 1501 {
1507 1502 exec_seen = 1;
1508 1503 close_user_mem();
1509 1504 }
1510 1505 }
1511 1506 else if ((act = valid_syscall(&a)) >= 0)
Show file before | Show file after | Raw diff | Download diff | Ignore whitespace Increase context Show commentsHide comments
@@ -1,108 +1,133
1 1 #!/usr/bin/env ruby
2 2
3 3 CORRECT_MARK = 'P'
4 4 INCORRECT_MARK = '-'
5 5 TIMEOUT_MARK = 'T'
6 6 RUN_ERROR_MARK = 'x'
7 7
8 8 def log(str='')
9 9 if ENV['TALKATIVE']!=nil
10 10 puts str
11 11 end
12 12 if ENV['GRADER_LOGGING']!=nil
13 13 log_fname = ENV['GRADER_LOGGING']
14 14 fp = File.open(log_fname,"a")
15 15 fp.puts("grade: #{Time.new.strftime("%H:%M")} #{str}")
16 16 fp.close
17 17 end
18 18 end
19 19
20 20 def char_comment(comment)
21 21 if comment =~ /[Ii]ncorrect/
22 22 INCORRECT_MARK
23 23 elsif comment =~ /[Cc]orrect/
24 24 CORRECT_MARK
25 25 elsif comment =~ /[Tt]ime/
26 26 TIMEOUT_MARK
27 27 elsif res = /^[Cc]omment:(.*)$/.match(comment)
28 28 res[1]
29 29 else
30 30 RUN_ERROR_MARK # these are run time errors
31 31 end
32 32 end
33 33
34 + def extract_time(t)
35 + puts "TIME: #{t}"
36 + if (result=/^(.*)r(.*)u(.*)s(.*)m/.match(t))
37 + {:real => result[1], :user => result[2], :sys => result[3], :mem => result[4]}
38 + else
39 + #{:real => 0, :user => 0, :sys => 0}
40 + #puts "ERROR READING RUNNING TIME: #{t}"
41 + raise "Error reading running time: #{t}"
42 + end
43 + end
44 +
34 45 problem_home = ENV['PROBLEM_HOME']
35 46 require "#{problem_home}/script/test_dsl.rb"
36 47 load "#{problem_home}/test_cases/all_tests.cfg"
37 48 problem = Problem.get_instance
38 49
39 50 if problem.well_formed? == false
40 51 log "The problem specification is not well formed."
41 52 exit(127)
42 53 end
43 54
44 55 all_score = 0
45 56 all_comment = ''
57 + peak_memory = -1
58 + max_runtime = -1
46 59 (1..(problem.runs.length-1)).each do |k|
47 60 log "grade run #{k}"
48 61 run = problem.runs[k]
49 62 run_score = nil
50 63 run_comment = ''
51 64 run_comment_short = ''
52 65 run.tests.each do |test_num|
53 66 result_file_name = "#{test_num}/result"
54 67 if not File.exists?(result_file_name)
55 68 run_comment += "result file for test #{test_num} not found\n"
56 69 run_comment_short += RUN_ERROR_MARK
57 70 log "Cannot find the file #{test_num}/result!"
58 71 else
59 72 result_file = File.new(result_file_name, "r")
60 73 result_file_lines = result_file.readlines
61 - if result_file_lines.length>=2
74 + if result_file_lines.length>=3
62 75 current_run_score = result_file_lines[1].to_i
63 76 run_comment += result_file_lines[0]
64 77 run_comment_short += char_comment(result_file_lines[0].chomp)
78 +
79 + #update max runtime & memory
80 + run_stat = extract_time result_file_lines[2]
81 + peak_memory = [peak_memory,run_stat[:mem].to_i].max
82 + max_runtime = [max_runtime,run_stat[:user].to_f + run_stat[:sys].to_f].max
65 83 else
66 84 current_run_score = 0
67 85 run_comment += "result file for test #{test_num} error\n"
68 86 run_comment_short += RUN_ERROR_MARK
69 87 log "Error in #{test_num}/result!"
70 88 end
71 89
72 90 # the score of this run should be the minimum of the score for
73 91 # each test case
74 92 if (run_score==nil) or (run_score>current_run_score)
75 93 run_score = current_run_score
76 94 end
77 95 result_file.close
78 96 end
79 97 end
80 98
81 99 run_result_file = File.new("result-#{k}", "w")
82 100 run_result_file.write run_score
83 101 run_result_file.write "\n"
84 102 run_result_file.close
85 103
86 104 run_comment_file = File.new("comment-#{k}", "w")
87 105 run_comment_file.write "#{run_comment}\n"
88 106 run_comment_file.close
89 107
90 108 all_score = all_score + run_score
91 109
92 110 # append comment for test run with many test cases
93 111 if run.tests.length > 1
94 112 run_comment_short = '[' + run_comment_short + ']'
95 113 end
96 114 all_comment += run_comment_short
97 115 end
98 116
99 117 result_file = File.new("result", "w")
100 118 result_file.write all_score
101 119 result_file.write "\n"
102 120 result_file.close
103 121
104 122 comment_file = File.new("comment", "w")
105 123 comment_file.write "#{all_comment}\n"
106 124 comment_file.close
107 125
108 - log "score = #{all_score} comment = #{all_comment}"
126 +
127 + File.open("run_stat","w") do |file|
128 + file.puts max_runtime
129 + file.puts peak_memory
130 + end
131 +
132 + log "score = #{all_score}\ncomment = #{all_comment}"
133 + log "max_runtime = #{max_runtime}\npeak_memory = #{peak_memory}"
You need to be logged in to leave comments. Login now