cafe-grader-judge-scripts Commit

cafe-grader-judge-scripts

Commit `r160:f0502be39e3b` public

parent | child

Description:

add run_stat checking

Commit status:

[Not Reviewed]

References:

algo

Diff options:

| | | |

Comments:

0 Commit comments 0 Inline Comments

Unresolved TODOs:

There are no unresolved TODOs

Add another comment

r160:f0502be39e3b - Fri, 29 Aug 2014 17:04:23 - 3 files changed: 48 inserted, 10 deleted

lib/submission_helper.rb ¶ +21 -3

               module Grader
                 class SubmissionRoomMaker
                   def initialize
                     @config = Grader::Configuration.get_instance
                   end
                   def produce_grading_room(submission)
                     user = submission.user
                     problem = submission.problem
                     grading_room = "#{@config.user_result_dir}/" +
                       "#{user.login}/#{problem.name}/#{submission.id}"
                     FileUtils.mkdir_p(grading_room)
                     grading_room
                   end
                   def find_problem_home(submission)
                     problem = submission.problem
                     "#{@config.problems_dir}/#{problem.name}"
                   end
                   def save_source(submission,source_name)
                     dir = self.produce_grading_room(submission)
                     f = File.open("#{dir}/#{source_name}","w")
                     f.write(submission.source)
                     f.close
                   end
                   def clean_up(submission)
                   end
                 end
                 class SubmissionReporter
                   def initialize(options={})
                     options = {:dry_run => false, :result_collector => nil}.merge(options)
                     @config = Grader::Configuration.get_instance
                     @dry_run = options[:dry_run]
                     @result_collector = options[:result_collector]
                   end
                   def report(sub,test_result_dir)
                     result = read_result(test_result_dir)
                     if @result_collector
                       @result_collector.save(sub,
                                              result)
                     end
                     save_result(sub,result)
                   end
                   def report_error(sub,msg)
                     save_result(sub,{:points => 0,
                                   :comment => "Grading error: #{msg}" })
                   end
                   protected
                   def read_result(test_result_dir)
                     cmp_msg_fname = "#{test_result_dir}/compiler_message"
                     if FileTest.exist?(cmp_msg_fname)
                       cmp_file = File.open(cmp_msg_fname)
                       cmp_msg = cmp_file.read
                       cmp_file.close
                     else
                       cmp_msg = ""
                     end
                     result_fname = "#{test_result_dir}/result"
                     comment_fname = "#{test_result_dir}/comment"
+            +       runstat_fname = "#{test_result_dir}/run_stat"
                     if FileTest.exist?(result_fname)
                       comment = ""
                       begin
                         result_file = File.open(result_fname)
                         result = result_file.readline.to_i
                         result_file.close
                       rescue
                         result = 0
                         comment = "error reading result file."
                       end
                       begin
                         comment_file = File.open(comment_fname)
                         comment += comment_file.readline.chomp
                         comment_file.close
                       rescue
                         comment += ""
                       end
-            -         return {:points => result,
+            +         begin
-            -           :comment => comment,
+            +           runstat_file = File.open(runstat_fname)
-            -           :cmp_msg => cmp_msg}
+            +           max_runtime = runstat_file.readline.to_f
+            +           peak_memory = runstat_file.readline.to_i
+            +         rescue
+            +           max_runtime = -1
+            +           peak_memory = -1
+            +         end
+            +
+            +
+            +         return {points: result,
+            +           comment: comment,
+            +           cmp_msg: cmp_msg,
+            +           max_runtime: max_runtime,
+            +           peak_memory: peak_memory
+            +         }
                     else
                       if FileTest.exist?("#{test_result_dir}/a.out")
                         return {:points => 0,
                           :comment => 'error during grading',
                           :cmp_msg => cmp_msg}
                       else
                         return {:points => 0,
                           :comment => 'compilation error',
                           :cmp_msg => cmp_msg}
                       end
                     end
                   end
                   def save_result(submission,result)
                     problem = submission.problem
                     submission.graded_at = Time.now.gmtime
                     points = result[:points]
                     submission.points = points
                     comment = @config.report_comment(result[:comment])
+            +       submission.peak_memory = result[:peak_memory]
+            +       submission.max_runtime =  result[:max_runtime]
+            +       submission.effective_code_length =submission.source.length
+            +
                     #
                     # TODO: FIX THIS MESSAGE
                     #
                     if problem == nil
                       submission.grader_comment = 'PASSED: ' + comment + '(problem is nil)'
                     elsif points == problem.full_score
                       #submission.grader_comment = 'PASSED: ' + comment
                       submission.grader_comment = comment
                     elsif result[:comment].chomp =~ /^[\[\]P]+$/
                       submission.grader_comment = 'PASSED: ' + comment + '(inconsistent score)'
                     else
                       #submission.grader_comment = 'FAILED: ' + comment
                       submission.grader_comment = comment
                     end
                     submission.compiler_message = result[:cmp_msg] or ''
                     if not @dry_run
                       submission.save
                     end
                   end
                 end
               end

std-script/box64-new.c ¶ -5

               }
               static void
               signal_alarm(int unused UNUSED)
               {
                 /* Time limit checks are synchronous, so we only schedule them there. */
                 timer_tick = 1;
                 alarm(1);
               }
               static void
               signal_int(int unused UNUSED)
               {
                 /* Interrupts are fatal, so no synchronization requirements. */
                 meta_printf("exitsig:%d\n", SIGINT);
                 err("SG: Interrupted");
               }
               #define PROC_BUF_SIZE 4096
               static void
               read_proc_file(char *buf, char *name, int *fdp)
               {
                 int c;
                 if (!*fdp)
                   {
                     sprintf(buf, "/proc/%d/%s", (int) box_pid, name);
                     *fdp = open(buf, O_RDONLY);
                     if (*fdp < 0)
               	die("open(%s): %m", buf);
                   }
                 lseek(*fdp, 0, SEEK_SET);
                 if ((c = read(*fdp, buf, PROC_BUF_SIZE-1)) < 0)
                   die("read on /proc/$pid/%s: %m", name);
                 if (c >= PROC_BUF_SIZE-1)
                   die("/proc/$pid/%s too long", name);
                 buf[c] = 0;
               }
               static void
               check_timeout(void)
               {
                 if (wall_timeout)
                   {
                     struct timeval now, wall;
                     int wall_ms;
                     gettimeofday(&now, NULL);
                     timersub(&now, &start_time, &wall);
                     wall_ms = wall.tv_sec*1000 + wall.tv_usec/1000;
                     if (wall_ms > wall_timeout)
                       err("TO: Time limit exceeded (wall clock)");
                     if (verbose > 1)
                       fprintf(stderr, "[wall time check: %d msec]\n", wall_ms);
                   }
                 if (timeout)
                   {
                     char buf[PROC_BUF_SIZE], *x;
                     int utime, stime, ms;
                     static int proc_stat_fd;
                     read_proc_file(buf, "stat", &proc_stat_fd);
                     x = buf;
                     while (*x && *x != ' ')
               	x++;
                     while (*x == ' ')
               	x++;
                     if (*x++ != '(')
               	die("proc stat syntax error 1");
                     while (*x && (*x != ')' || x[1] != ' '))
               	x++;
                     while (*x == ')' || *x == ' ')
               	x++;
                     if (sscanf(x, "%*c %*d %*d %*d %*d %*d %*d %*d %*d %*d %*d %d %d", &utime, &stime) != 2)
               	die("proc stat syntax error 2");
                     ms = (utime + stime) * 1000 / ticks_per_sec;
                     if (verbose > 1)
               	fprintf(stderr, "[time check: %d msec]\n", ms);
                     if (ms > timeout && ms > extra_timeout)
               	err("TO: Time limit exceeded");
                   }
               }
               static void
               sample_mem_peak(void)
               {
                 /*
                  *  We want to find out the peak memory usage of the process, which is
                  *  maintained by the kernel, but unforunately it gets lost when the
                  *  process exits (it is not reported in struct rusage). Therefore we
                  *  have to sample it whenever we suspect that the process is about
                  *  to exit.
                  */
                 char buf[PROC_BUF_SIZE], *x;
                 static int proc_status_fd;
                 read_proc_file(buf, "status", &proc_status_fd);
                 x = buf;
                 while (*x)
                   {
                     char *key = x;
                     while (*x && *x != ':' && *x != '\n')
               	x++;
                     if (!*x || *x == '\n')
               	break;
                     *x++ = 0;
                     while (*x == ' ' || *x == '\t')
               	x++;
                     char *val = x;
                     while (*x && *x != '\n')
               	x++;
                     if (!*x)
               	break;
                     *x++ = 0;
                     if (!strcmp(key, "VmPeak"))
               	{
               	  int peak = atoi(val);
               	  if (peak > mem_peak_kb)
               	    mem_peak_kb = peak;
               	}
                   }
                 if (verbose > 1)
                   msg("[mem-peak: %u KB]\n", mem_peak_kb);
               }
               static void
               boxkeeper(void)
               {
                 int syscall_count = (filter_syscalls ? 0 : 1);
                 struct sigaction sa;
                 is_ptraced = 1;
                 bzero(&sa, sizeof(sa));
                 sa.sa_handler = signal_int;
                 sigaction(SIGINT, &sa, NULL);
                 gettimeofday(&start_time, NULL);
                 ticks_per_sec = sysconf(_SC_CLK_TCK);
                 if (ticks_per_sec <= 0)
                   die("Invalid ticks_per_sec!");
                 if (timeout || wall_timeout)
                   {
                     sa.sa_handler = signal_alarm;
                     sigaction(SIGALRM, &sa, NULL);
                     alarm(1);
                   }
                 for(;;)
                   {
                     struct rusage rus;
                     int stat;
                     pid_t p;
                     if (timer_tick)
               	{
               	  check_timeout();
               	  timer_tick = 0;
               	}
                     p = wait4(box_pid, &stat, WUNTRACED, &rus);
                     if (p < 0)
               	{
               	  if (errno == EINTR)
               	    continue;
               	  die("wait4: %m");
               	}
                     if (p != box_pid)
               	die("wait4: unknown pid %d exited!", p);
                     if (WIFEXITED(stat))
               	{
               	  box_pid = 0;
               	  final_stats(&rus);
               	  if (WEXITSTATUS(stat))
               	    {
               	      if (syscall_count)
               		{
               		  meta_printf("exitcode:%d\n", WEXITSTATUS(stat));
               		  err("RE: Exited with error status %d", WEXITSTATUS(stat));
               		}
               	      else
               		{
               		  // Internal error happened inside the child process and it has been already reported.
               		  box_exit(2);
               		}
               	    }
               	  if (timeout && total_ms > timeout)
               	    err("TO: Time limit exceeded");
               	  if (wall_timeout && wall_ms > wall_timeout)
               	    err("TO: Time limit exceeded (wall clock)");
               	  flush_line();
                   fprintf(stderr,"OK\n");
-            -         print_running_stat(
-            -               (double)wall_ms/1000,
-            -               (double)total_ms/1000,
-            -               (double)sys_ms/1000,
-            -               (mem_peak_kb + 1023) / 1024);
               	  box_exit(0);
               	}
                     if (WIFSIGNALED(stat))
               	{
               	  box_pid = 0;
               	  meta_printf("exitsig:%d\n", WTERMSIG(stat));
               	  final_stats(&rus);
               	  err("SG: Caught fatal signal %d%s", WTERMSIG(stat), (syscall_count ? "" : " during startup"));
               	}
                     if (WIFSTOPPED(stat))
               	{
               	  int sig = WSTOPSIG(stat);
               	  if (sig == SIGTRAP)
               	    {
               	      if (verbose > 2)
               		msg("[ptrace status %08x] ", stat);
               	      static int stop_count;
               	      if (!stop_count++)		/* Traceme request */
               		msg(">> Traceme request caught\n");
               	      else
               		err("SG: Breakpoint");
               	      ptrace(PTRACE_SYSCALL, box_pid, 0, 0);
               	    }
               	  else if (sig == (SIGTRAP | 0x80))
               	    {
               	      if (verbose > 2)
               		msg("[ptrace status %08x] ", stat);
               	      struct syscall_args a;
               	      static unsigned int sys_tick, last_act;
               	      static arg_t last_sys;
               	      if (++sys_tick & 1)		/* Syscall entry */
               		{
               		  char namebuf[32];
               		  int act;
               		  get_syscall_args(&a, 0);
               		  arg_t sys = a.sys;
               		  msg(">> Syscall %-12s (%08jx,%08jx,%08jx) ", syscall_name(sys, namebuf), (intmax_t) a.arg1, (intmax_t) a.arg2, (intmax_t) a.arg3);
               		  if (!exec_seen)
               		    {
               		      msg("[master] ");
               		      if (sys == NATIVE_NR_execve)
               			{
               			  exec_seen = 1;
               			  close_user_mem();
               			}
               		    }
               		  else if ((act = valid_syscall(&a)) >= 0)
               		    {
               		      last_act = act;
               		      syscall_count++;
               		      if (act & A_SAMPLE_MEM)
               			sample_mem_peak();
               		    }
               		  else
               		    {
               		      /*
               		       * Unfortunately, PTRACE_KILL kills _after_ the syscall completes,
               		       * so we have to change it to something harmless (e.g., an undefined
               		       * syscall) and make the program continue.
               		       */
               		      set_syscall_nr(&a, ~(arg_t)0);
               		      err("FO: Forbidden syscall %s", syscall_name(sys, namebuf));
               		    }
               		  last_sys = sys;
               		}
               	      else					/* Syscall return */
               		{
               		  get_syscall_args(&a, 1);
               		  if (a.sys == ~(arg_t)0)
               		    {
               		      /* Some syscalls (sigreturn et al.) do not return a value */
               		      if (!(last_act & A_NO_RETVAL))
               			err("XX: Syscall does not return, but it should");
               		    }
               		  else
               		    {
               		      if (a.sys != last_sys)
               			err("XX: Mismatched syscall entry/exit");
               		    }
               		  if (last_act & A_NO_RETVAL)
               		    msg("= ?\n");
               		  else
               		    msg("= %jd\n", (intmax_t) a.result);
               		}
               	      ptrace(PTRACE_SYSCALL, box_pid, 0, 0);
               	    }
               	  else if (sig == SIGSTOP)
               	    {
               	      msg(">> SIGSTOP\n");
               	      if (ptrace(PTRACE_SETOPTIONS, box_pid, NULL, (void *) PTRACE_O_TRACESYSGOOD) < 0)
               		die("ptrace(PTRACE_SETOPTIONS): %m");
               	      ptrace(PTRACE_SYSCALL, box_pid, 0, 0);
               	    }
               	  else if (sig != SIGXCPU && sig != SIGXFSZ)
               	    {
               	      msg(">> Signal %d\n", sig);
               	      sample_mem_peak();			/* Signal might be fatal, so update mem-peak */
               	      ptrace(PTRACE_SYSCALL, box_pid, 0, sig);
               	    }
               	  else
               	    {
               	      meta_printf("exitsig:%d", sig);
               	      err("SG: Received signal %d", sig);
               	    }
               	}
                     else
               	die("wait4: unknown status %x, giving up!", stat);
                   }
               }
               static void
               box_inside(int argc, char **argv)
               {
                 struct rlimit rl;
                 char *args[argc+1];
                 memcpy(args, argv, argc * sizeof(char *));
                 args[argc] = NULL;
                 if (set_cwd && chdir(set_cwd))
                   die("chdir: %m");
                 if (redir_stdin)
                   {
                     close(0);
                     if (open(redir_stdin, O_RDONLY) != 0)
               	die("open(\"%s\"): %m", redir_stdin);
                   }
                 if (redir_stdout)
                   {
                     close(1);
                     if (open(redir_stdout, O_WRONLY | O_CREAT | O_TRUNC, 0666) != 1)
               	die("open(\"%s\"): %m", redir_stdout);
                   }
                 if (redir_stderr)
                   {
                     close(2);
                     if (open(redir_stderr, O_WRONLY | O_CREAT | O_TRUNC, 0666) != 2)
               	die("open(\"%s\"): %m", redir_stderr);
                   }
                 else
                   dup2(1, 2);
                 setpgrp();
                 if (memory_limit)
                   {
                     rl.rlim_cur = rl.rlim_max = memory_limit * 1024;
                     if (setrlimit(RLIMIT_AS, &rl) < 0)
               	die("setrlimit(RLIMIT_AS): %m");
                   }
                 rl.rlim_cur = rl.rlim_max = (stack_limit ? (rlim_t)stack_limit * 1024 : RLIM_INFINITY);
                 if (setrlimit(RLIMIT_STACK, &rl) < 0)
                   die("setrlimit(RLIMIT_STACK): %m");
                 rl.rlim_cur = rl.rlim_max = 64;
                 if (setrlimit(RLIMIT_NOFILE, &rl) < 0)
                   die("setrlimit(RLIMIT_NOFILE): %m");
                 char **env = setup_environment();
                 if (filter_syscalls)
                   {
                     if (ptrace(PTRACE_TRACEME) < 0)
               	die("ptrace(PTRACE_TRACEME): %m");
                     /* Trick: Make sure that we are stopped until the boxkeeper wakes up. */
                     raise(SIGSTOP);
                   }
                 execve(args[0], args, env);
                 die("execve(\"%s\"): %m", args[0]);
               }
               static void
               usage(void)
               {
                 fprintf(stderr, "Invalid arguments!\n");
                 printf("\
               Usage: box [<options>] -- <command> <arguments>\n\
               \n\
               Options:\n\
               -a <level>\tSet file access level (0=none, 1=cwd, 2=/etc,/lib,..., 3=whole fs, 9=no checks; needs -f)\n\
               -c <dir>\tChange directory to <dir> first\n\
               -e\t\tInherit full environment of the parent process\n\
               -E <var>\tInherit the environment variable <var> from the parent process\n\
               -E <var>=<val>\tSet the environment variable <var> to <val>; unset it if <var> is empty\n\
               -f\t\tFilter system calls (-ff=very restricted)\n\
               -i <file>\tRedirect stdin from <file>\n\
               -k <size>\tLimit stack size to <size> KB (default: 0=unlimited)\n\
               -m <size>\tLimit address space to <size> KB\n\
               -M <file>\tOutput process information to <file> (name:value)\n\
               -o <file>\tRedirect stdout to <file>\n\
               -p <path>\tPermit access to the specified path (or subtree if it ends with a `/')\n\
               -p <path>=<act>\tDefine action for the specified path (<act>=yes/no)\n\
               -r <file>\tRedirect stderr to <file>\n\

std-script/grade ¶ +27 -2

               #!/usr/bin/env ruby
               CORRECT_MARK = 'P'
               INCORRECT_MARK = '-'
               TIMEOUT_MARK = 'T'
               RUN_ERROR_MARK = 'x'
               def log(str='')
                 if ENV['TALKATIVE']!=nil
                   puts str
                 end
                 if ENV['GRADER_LOGGING']!=nil
                   log_fname = ENV['GRADER_LOGGING']
                   fp = File.open(log_fname,"a")
                   fp.puts("grade: #{Time.new.strftime("%H:%M")} #{str}")
                   fp.close
                 end
               end
               def char_comment(comment)
                 if comment =~ /[Ii]ncorrect/
                   INCORRECT_MARK
                 elsif comment =~ /[Cc]orrect/
                   CORRECT_MARK
                 elsif comment =~ /[Tt]ime/
                   TIMEOUT_MARK
                 elsif res = /^[Cc]omment:(.*)$/.match(comment)
                   res[1]
                 else
                   RUN_ERROR_MARK  # these are run time errors
                 end
               end
+            + def extract_time(t)
+            +   puts "TIME: #{t}"
+            +   if (result=/^(.*)r(.*)u(.*)s(.*)m/.match(t))
+            +     {:real => result[1], :user => result[2], :sys => result[3], :mem => result[4]}
+            +   else
+            +     #{:real => 0, :user => 0, :sys => 0}
+            +     #puts "ERROR READING RUNNING TIME: #{t}"
+            +     raise "Error reading running time: #{t}"
+            +   end
+            + end
+            +
               problem_home = ENV['PROBLEM_HOME']
               require "#{problem_home}/script/test_dsl.rb"
               load "#{problem_home}/test_cases/all_tests.cfg"
               problem = Problem.get_instance
               if problem.well_formed? == false
                 log "The problem specification is not well formed."
                 exit(127)
               end
               all_score = 0
               all_comment = ''
+            + peak_memory = -1
+            + max_runtime = -1
               (1..(problem.runs.length-1)).each do |k|
                 log "grade run #{k}"
                 run = problem.runs[k]
                 run_score = nil
                 run_comment = ''
                 run_comment_short = ''
                 run.tests.each do |test_num|
                   result_file_name = "#{test_num}/result"
                   if not File.exists?(result_file_name)
                     run_comment += "result file for test #{test_num} not found\n"
                     run_comment_short += RUN_ERROR_MARK
                     log "Cannot find the file #{test_num}/result!"
                   else
                     result_file = File.new(result_file_name, "r")
                     result_file_lines = result_file.readlines
-            -       if result_file_lines.length>=2
+            +       if result_file_lines.length>=3
                       current_run_score = result_file_lines[1].to_i
                       run_comment += result_file_lines[0]
                       run_comment_short += char_comment(result_file_lines[0].chomp)
+            +
+            +         #update max runtime & memory
+            +         run_stat = extract_time result_file_lines[2]
+            +         peak_memory = [peak_memory,run_stat[:mem].to_i].max
+            +         max_runtime = [max_runtime,run_stat[:user].to_f + run_stat[:sys].to_f].max
                     else
                       current_run_score = 0
                       run_comment += "result file for test #{test_num} error\n"
                       run_comment_short += RUN_ERROR_MARK
                       log "Error in #{test_num}/result!"
                     end
                     # the score of this run should be the minimum of the score for
                     #   each test case
                     if (run_score==nil) or (run_score>current_run_score)
                       run_score = current_run_score
                     end
                     result_file.close
                   end
                 end
                 run_result_file = File.new("result-#{k}", "w")
                 run_result_file.write run_score
                 run_result_file.write "\n"
                 run_result_file.close
                 run_comment_file = File.new("comment-#{k}", "w")
                 run_comment_file.write "#{run_comment}\n"
                 run_comment_file.close
                 all_score = all_score + run_score
                 # append comment for test run with many test cases
                 if run.tests.length > 1
                   run_comment_short = '[' + run_comment_short + ']'
                 end
                 all_comment += run_comment_short
               end
               result_file = File.new("result", "w")
               result_file.write all_score
               result_file.write "\n"
               result_file.close
               comment_file = File.new("comment", "w")
               comment_file.write "#{all_comment}\n"
               comment_file.close
-            - log "score = #{all_score} comment = #{all_comment}"
+            + File.open("run_stat","w") do |file|
+            +   file.puts max_runtime
+            +   file.puts peak_memory
+            + end
+            +
+            + log "score = #{all_score}\ncomment = #{all_comment}"
+            + log "max_runtime = #{max_runtime}\npeak_memory = #{peak_memory}"

Write
Preview

You need to be logged in to leave comments. Login now

	Site-wide shortcuts
/	Use quick search box
g h	Goto home page
g g	Goto my private gists page
g G	Goto my public gists page
n r	New repository page
n g	New gist page

	Repositories
g s	Goto summary page
g c	Goto changelog page
g f	Goto files page
g F	Goto files page with file search activated
g p	Goto pull requests page
g o	Goto repository settings
g O	Goto repository permissions settings

Sign in to your account

Author

r160:f0502be39e3b - Fri, 29 Aug 2014 17:04:23 - 3 files changed: 48 inserted, 10 deleted

Sign in to your account

Commit r160:f0502be39e3b public

Author

r160:f0502be39e3b - Fri, 29 Aug 2014 17:04:23 - 3 files changed: 48 inserted, 10 deleted

Commit `r160:f0502be39e3b` public