cafe-grader-judge-scripts Commit

cafe-grader-judge-scripts

Commit `r160:f0502be39e3b` public

parent | child

Description:

add run_stat checking

Commit status:

[Not Reviewed]

References:

algo

Diff options:

| | | |

Comments:

0 Commit comments 0 Inline Comments

Unresolved TODOs:

There are no unresolved TODOs

Add another comment

r160:f0502be39e3b - Fri, 29 Aug 2014 17:04:23 - 3 files changed: 48 inserted, 10 deleted

lib/submission_helper.rb ¶ +21 -3

               module Grader
                 class SubmissionRoomMaker
                   def initialize
                     @config = Grader::Configuration.get_instance
                   end
                   def produce_grading_room(submission)
                     user = submission.user
                     problem = submission.problem
                     grading_room = "#{@config.user_result_dir}/" +
                       "#{user.login}/#{problem.name}/#{submission.id}"
                     FileUtils.mkdir_p(grading_room)
                     grading_room
                   end
                   def find_problem_home(submission)
                     problem = submission.problem
                     "#{@config.problems_dir}/#{problem.name}"
                   end
                   def save_source(submission,source_name)
                     dir = self.produce_grading_room(submission)
                     f = File.open("#{dir}/#{source_name}","w")
                     f.write(submission.source)
                     f.close
                   end
                   def clean_up(submission)
                   end
                 end
                 class SubmissionReporter
                   def initialize(options={})
                     options = {:dry_run => false, :result_collector => nil}.merge(options)
                     @config = Grader::Configuration.get_instance
                     @dry_run = options[:dry_run]
                     @result_collector = options[:result_collector]
                   end
                   def report(sub,test_result_dir)
                     result = read_result(test_result_dir)
                     if @result_collector
                       @result_collector.save(sub,
                                              result)
                     end
                     save_result(sub,result)
                   end
                   def report_error(sub,msg)
                     save_result(sub,{:points => 0,
                                   :comment => "Grading error: #{msg}" })
                   end
                   protected
                   def read_result(test_result_dir)
                     cmp_msg_fname = "#{test_result_dir}/compiler_message"
                     if FileTest.exist?(cmp_msg_fname)
                       cmp_file = File.open(cmp_msg_fname)
                       cmp_msg = cmp_file.read
                       cmp_file.close
                     else
                       cmp_msg = ""
                     end
                     result_fname = "#{test_result_dir}/result"
                     comment_fname = "#{test_result_dir}/comment"
+            +       runstat_fname = "#{test_result_dir}/run_stat"
                     if FileTest.exist?(result_fname)
                       comment = ""
                       begin
                         result_file = File.open(result_fname)
                         result = result_file.readline.to_i
                         result_file.close
                       rescue
                         result = 0
                         comment = "error reading result file."
                       end
                       begin
                         comment_file = File.open(comment_fname)
                         comment += comment_file.readline.chomp
                         comment_file.close
                       rescue
                         comment += ""
                       end
-            -         return {:points => result,
+            +         begin
-            -           :comment => comment,
+            +           runstat_file = File.open(runstat_fname)
-            -           :cmp_msg => cmp_msg}
+            +           max_runtime = runstat_file.readline.to_f
+            +           peak_memory = runstat_file.readline.to_i
+            +         rescue
+            +           max_runtime = -1
+            +           peak_memory = -1
+            +         end
+            +
+            +
+            +         return {points: result,
+            +           comment: comment,
+            +           cmp_msg: cmp_msg,
+            +           max_runtime: max_runtime,
+            +           peak_memory: peak_memory
+            +         }
                     else
                       if FileTest.exist?("#{test_result_dir}/a.out")
                         return {:points => 0,
                           :comment => 'error during grading',
                           :cmp_msg => cmp_msg}
                       else
                         return {:points => 0,
                           :comment => 'compilation error',
                           :cmp_msg => cmp_msg}
                       end
                     end
                   end
                   def save_result(submission,result)
                     problem = submission.problem
                     submission.graded_at = Time.now.gmtime
                     points = result[:points]
                     submission.points = points
                     comment = @config.report_comment(result[:comment])
+            +       submission.peak_memory = result[:peak_memory]
+            +       submission.max_runtime =  result[:max_runtime]
+            +       submission.effective_code_length =submission.source.length
+            +
                     #
                     # TODO: FIX THIS MESSAGE
                     #
                     if problem == nil
                       submission.grader_comment = 'PASSED: ' + comment + '(problem is nil)'
                     elsif points == problem.full_score
                       #submission.grader_comment = 'PASSED: ' + comment
                       submission.grader_comment = comment
                     elsif result[:comment].chomp =~ /^[\[\]P]+$/
                       submission.grader_comment = 'PASSED: ' + comment + '(inconsistent score)'
                     else
                       #submission.grader_comment = 'FAILED: ' + comment
                       submission.grader_comment = comment
                     end
                     submission.compiler_message = result[:cmp_msg] or ''
                     if not @dry_run
                       submission.save
                     end
                   end
                 end
               end

std-script/box64-new.c ¶ -5

               	{
               	case 0x050f:
               	  break;
               	case 0x80cd:
               	  err("FO: Forbidden 32-bit syscall in 64-bit mode");
               	default:
               	  err("XX: Unknown syscall instruction %04x", instr);
               	}
                     sys_type = 64;
                     break;
                   default:
                     err("XX: Unknown code segment %04jx", (intmax_t) a->user.regs.cs);
                   }
               #ifdef CONFIG_BOX_USER_AMD64
                 if (sys_type != 64)
                   err("FO: Forbidden %d-bit mode syscall", sys_type);
               #else
                 if (sys_type != (exec_seen ? 32 : 64))
                   err("FO: Forbidden %d-bit mode syscall", sys_type);
               #endif
                 if (sys_type == 32)
                   {
                     a->arg1 = a->user.regs.rbx;
                     a->arg2 = a->user.regs.rcx;
                     a->arg3 = a->user.regs.rdx;
                   }
                 else
                   {
                     a->arg1 = a->user.regs.rdi;
                     a->arg2 = a->user.regs.rsi;
                     a->arg3 = a->user.regs.rdx;
                   }
               }
               static void
               set_syscall_nr(struct syscall_args *a, arg_t sys)
               {
                 a->sys = sys;
                 a->user.regs.orig_rax = sys;
                 if (ptrace(PTRACE_SETREGS, box_pid, NULL, &a->user) < 0)
                   die("ptrace(PTRACE_SETREGS): %m");
               }
               static void
               sanity_check(void)
               {
               }
               #else
               static void
               get_syscall_args(struct syscall_args *a, int is_exit UNUSED)
               {
                 if (ptrace(PTRACE_GETREGS, box_pid, NULL, &a->user) < 0)
                   die("ptrace(PTRACE_GETREGS): %m");
                 a->sys = a->user.regs.orig_eax;
                 a->arg1 = a->user.regs.ebx;
                 a->arg2 = a->user.regs.ecx;
                 a->arg3 = a->user.regs.edx;
                 a->result = a->user.regs.eax;
               }
               static void
               set_syscall_nr(struct syscall_args *a, arg_t sys)
               {
                 a->sys = sys;
                 a->user.regs.orig_eax = sys;
                 if (ptrace(PTRACE_SETREGS, box_pid, NULL, &a->user) < 0)
                   die("ptrace(PTRACE_SETREGS): %m");
               }
               static void
               sanity_check(void)
               {
               #if !defined(CONFIG_BOX_ALLOW_INSECURE)
                 struct utsname uts;
                 if (uname(&uts) < 0)
                   die("uname() failed: %m");
                 if (!strcmp(uts.machine, "x86_64"))
                   die("Running 32-bit sandbox on 64-bit kernels is inherently unsafe. Please get a 64-bit version.");
               #endif
               }
               #endif
               /*** Syscall checks ***/
               static void
               valid_filename(arg_t addr)
               {
                 char namebuf[4096], *p, *end;
                 if (!file_access)
                   err("FA: File access forbidden");
                 if (file_access >= 9)
                   return;
                 p = end = namebuf;
                 do
                   {
                     if (p >= end)
               	{
               	  int remains = PAGE_SIZE - (addr & (PAGE_SIZE-1));
               	  int l = namebuf + sizeof(namebuf) - end;
               	  if (l > remains)
               	    l = remains;
               	  if (!l)
               	    err("FA: Access to file with name too long");
               	  remains = read_user_mem(addr, end, l);
               	  if (remains < 0)
               	    die("read(mem): %m");
               	  if (!remains)
               	    err("FA: Access to file with name out of memory");
               	  end += remains;
               	  addr += remains;
               	}
                   }
                 while (*p++);
                 msg("[%s] ", namebuf);
                 if (file_access >= 3)
                   return;
                 // Everything in current directory is permitted
                 if (!strchr(namebuf, '/') && strcmp(namebuf, ".."))
                   return;
                 // ".." anywhere in the path is forbidden
                 enum action act = A_DEFAULT;
                 if (strstr(namebuf, ".."))
                   act = A_NO;
                 // Scan user rules
                 for (struct path_rule *r = user_path_rules; r && !act; r=r->next)
                   act = match_path_rule(r, namebuf);
                 // Scan built-in rules
                 if (file_access >= 2)
                   for (int i=0; i<ARRAY_SIZE(default_path_rules) && !act; i++)
                     act = match_path_rule(&default_path_rules[i], namebuf);
                 if (act != A_YES)
                   err("FA: Forbidden access to file `%s'", namebuf);
               }
               // Check syscall. If invalid, return -1, otherwise return the action mask.
               static int
               valid_syscall(struct syscall_args *a)
               {
                 unsigned int sys = a->sys;
                 unsigned int act = (sys < NUM_ACTIONS) ? syscall_action[sys] : A_DEFAULT;
                 if (act & A_LIBERAL)
                   {
                     if (filter_syscalls != 1)
                       act = A_DEFAULT;
                   }
                 switch (act & A_ACTION_MASK)
                   {
                   case A_YES:
                     return act;
                   case A_NO:
                     return -1;
                   case A_FILENAME:
                     valid_filename(a->arg1);
                     return act;
                   default: ;
                   }
                 switch (sys)
                   {
                   case __NR_kill:
                     if (a->arg1 == (arg_t) box_pid)
               	{
               	  meta_printf("exitsig:%d\n", (int) a->arg2);
               	  err("SG: Committed suicide by signal %d", (int) a->arg2);
               	}
                     return -1;
                   case __NR_tgkill:
                     if (a->arg1 == (arg_t) box_pid && a->arg2 == (arg_t) box_pid)
               	{
               	  meta_printf("exitsig:%d\n", (int) a->arg3);
               	  err("SG: Committed suicide by signal %d", (int) a->arg3);
               	}
                     return -1;
                   default:
                     return -1;
                   }
               }
               static void
               signal_alarm(int unused UNUSED)
               {
                 /* Time limit checks are synchronous, so we only schedule them there. */
                 timer_tick = 1;
                 alarm(1);
               }
               static void
               signal_int(int unused UNUSED)
               {
                 /* Interrupts are fatal, so no synchronization requirements. */
                 meta_printf("exitsig:%d\n", SIGINT);
                 err("SG: Interrupted");
               }
               #define PROC_BUF_SIZE 4096
               static void
               read_proc_file(char *buf, char *name, int *fdp)
               {
                 int c;
                 if (!*fdp)
                   {
                     sprintf(buf, "/proc/%d/%s", (int) box_pid, name);
                     *fdp = open(buf, O_RDONLY);
                     if (*fdp < 0)
               	die("open(%s): %m", buf);
                   }
                 lseek(*fdp, 0, SEEK_SET);
                 if ((c = read(*fdp, buf, PROC_BUF_SIZE-1)) < 0)
                   die("read on /proc/$pid/%s: %m", name);
                 if (c >= PROC_BUF_SIZE-1)
                   die("/proc/$pid/%s too long", name);
                 buf[c] = 0;
               }
               static void
               check_timeout(void)
               {
                 if (wall_timeout)
                   {
                     struct timeval now, wall;
                     int wall_ms;
                     gettimeofday(&now, NULL);
                     timersub(&now, &start_time, &wall);
                     wall_ms = wall.tv_sec*1000 + wall.tv_usec/1000;
                     if (wall_ms > wall_timeout)
                       err("TO: Time limit exceeded (wall clock)");
                     if (verbose > 1)
                       fprintf(stderr, "[wall time check: %d msec]\n", wall_ms);
                   }
                 if (timeout)
                   {
                     char buf[PROC_BUF_SIZE], *x;
                     int utime, stime, ms;
                     static int proc_stat_fd;
                     read_proc_file(buf, "stat", &proc_stat_fd);
                     x = buf;
                     while (*x && *x != ' ')
               	x++;
                     while (*x == ' ')
               	x++;
                     if (*x++ != '(')
               	die("proc stat syntax error 1");
                     while (*x && (*x != ')' || x[1] != ' '))
               	x++;
                     while (*x == ')' || *x == ' ')
               	x++;
                     if (sscanf(x, "%*c %*d %*d %*d %*d %*d %*d %*d %*d %*d %*d %d %d", &utime, &stime) != 2)
               	die("proc stat syntax error 2");
                     ms = (utime + stime) * 1000 / ticks_per_sec;
                     if (verbose > 1)
               	fprintf(stderr, "[time check: %d msec]\n", ms);
                     if (ms > timeout && ms > extra_timeout)
               	err("TO: Time limit exceeded");
                   }
               }
               static void
               sample_mem_peak(void)
               {
                 /*
                  *  We want to find out the peak memory usage of the process, which is
                  *  maintained by the kernel, but unforunately it gets lost when the
                  *  process exits (it is not reported in struct rusage). Therefore we
                  *  have to sample it whenever we suspect that the process is about
                  *  to exit.
                  */
                 char buf[PROC_BUF_SIZE], *x;
                 static int proc_status_fd;
                 read_proc_file(buf, "status", &proc_status_fd);
                 x = buf;
                 while (*x)
                   {
                     char *key = x;
                     while (*x && *x != ':' && *x != '\n')
               	x++;
                     if (!*x || *x == '\n')
               	break;
                     *x++ = 0;
                     while (*x == ' ' || *x == '\t')
               	x++;
                     char *val = x;
                     while (*x && *x != '\n')
               	x++;
                     if (!*x)
               	break;
                     *x++ = 0;
                     if (!strcmp(key, "VmPeak"))
               	{
               	  int peak = atoi(val);
               	  if (peak > mem_peak_kb)
               	    mem_peak_kb = peak;
               	}
                   }
                 if (verbose > 1)
                   msg("[mem-peak: %u KB]\n", mem_peak_kb);
               }
               static void
               boxkeeper(void)
               {
                 int syscall_count = (filter_syscalls ? 0 : 1);
                 struct sigaction sa;
                 is_ptraced = 1;
                 bzero(&sa, sizeof(sa));
                 sa.sa_handler = signal_int;
                 sigaction(SIGINT, &sa, NULL);
                 gettimeofday(&start_time, NULL);
                 ticks_per_sec = sysconf(_SC_CLK_TCK);
                 if (ticks_per_sec <= 0)
                   die("Invalid ticks_per_sec!");
                 if (timeout || wall_timeout)
                   {
                     sa.sa_handler = signal_alarm;
                     sigaction(SIGALRM, &sa, NULL);
                     alarm(1);
                   }
                 for(;;)
                   {
                     struct rusage rus;
                     int stat;
                     pid_t p;
                     if (timer_tick)
               	{
               	  check_timeout();
               	  timer_tick = 0;
               	}
                     p = wait4(box_pid, &stat, WUNTRACED, &rus);
                     if (p < 0)
               	{
               	  if (errno == EINTR)
               	    continue;
               	  die("wait4: %m");
               	}
                     if (p != box_pid)
               	die("wait4: unknown pid %d exited!", p);
                     if (WIFEXITED(stat))
               	{
               	  box_pid = 0;
               	  final_stats(&rus);
               	  if (WEXITSTATUS(stat))
               	    {
               	      if (syscall_count)
               		{
               		  meta_printf("exitcode:%d\n", WEXITSTATUS(stat));
               		  err("RE: Exited with error status %d", WEXITSTATUS(stat));
               		}
               	      else
               		{
               		  // Internal error happened inside the child process and it has been already reported.
               		  box_exit(2);
               		}
               	    }
               	  if (timeout && total_ms > timeout)
               	    err("TO: Time limit exceeded");
               	  if (wall_timeout && wall_ms > wall_timeout)
               	    err("TO: Time limit exceeded (wall clock)");
               	  flush_line();
                   fprintf(stderr,"OK\n");
-            -         print_running_stat(
-            -               (double)wall_ms/1000,
-            -               (double)total_ms/1000,
-            -               (double)sys_ms/1000,
-            -               (mem_peak_kb + 1023) / 1024);
               	  box_exit(0);
               	}
                     if (WIFSIGNALED(stat))
               	{
               	  box_pid = 0;
               	  meta_printf("exitsig:%d\n", WTERMSIG(stat));
               	  final_stats(&rus);
               	  err("SG: Caught fatal signal %d%s", WTERMSIG(stat), (syscall_count ? "" : " during startup"));
               	}
                     if (WIFSTOPPED(stat))
               	{
               	  int sig = WSTOPSIG(stat);
               	  if (sig == SIGTRAP)
               	    {
               	      if (verbose > 2)
               		msg("[ptrace status %08x] ", stat);
               	      static int stop_count;
               	      if (!stop_count++)		/* Traceme request */
               		msg(">> Traceme request caught\n");
               	      else
               		err("SG: Breakpoint");
               	      ptrace(PTRACE_SYSCALL, box_pid, 0, 0);
               	    }
               	  else if (sig == (SIGTRAP | 0x80))
               	    {
               	      if (verbose > 2)
               		msg("[ptrace status %08x] ", stat);
               	      struct syscall_args a;
               	      static unsigned int sys_tick, last_act;
               	      static arg_t last_sys;
               	      if (++sys_tick & 1)		/* Syscall entry */
               		{
               		  char namebuf[32];
               		  int act;
               		  get_syscall_args(&a, 0);
               		  arg_t sys = a.sys;
               		  msg(">> Syscall %-12s (%08jx,%08jx,%08jx) ", syscall_name(sys, namebuf), (intmax_t) a.arg1, (intmax_t) a.arg2, (intmax_t) a.arg3);
               		  if (!exec_seen)
               		    {
               		      msg("[master] ");
               		      if (sys == NATIVE_NR_execve)
               			{
               			  exec_seen = 1;
               			  close_user_mem();
               			}
               		    }
               		  else if ((act = valid_syscall(&a)) >= 0)
               		    {
               		      last_act = act;
               		      syscall_count++;
               		      if (act & A_SAMPLE_MEM)
               			sample_mem_peak();
               		    }
               		  else
               		    {
               		      /*
               		       * Unfortunately, PTRACE_KILL kills _after_ the syscall completes,
               		       * so we have to change it to something harmless (e.g., an undefined
               		       * syscall) and make the program continue.
               		       */
               		      set_syscall_nr(&a, ~(arg_t)0);
               		      err("FO: Forbidden syscall %s", syscall_name(sys, namebuf));
               		    }
               		  last_sys = sys;
               		}
               	      else					/* Syscall return */
               		{
               		  get_syscall_args(&a, 1);
               		  if (a.sys == ~(arg_t)0)
               		    {
               		      /* Some syscalls (sigreturn et al.) do not return a value */
               		      if (!(last_act & A_NO_RETVAL))
               			err("XX: Syscall does not return, but it should");
               		    }
               		  else
               		    {
               		      if (a.sys != last_sys)
               			err("XX: Mismatched syscall entry/exit");
               		    }
               		  if (last_act & A_NO_RETVAL)
               		    msg("= ?\n");
               		  else
               		    msg("= %jd\n", (intmax_t) a.result);
               		}
               	      ptrace(PTRACE_SYSCALL, box_pid, 0, 0);
               	    }
               	  else if (sig == SIGSTOP)
               	    {
               	      msg(">> SIGSTOP\n");
               	      if (ptrace(PTRACE_SETOPTIONS, box_pid, NULL, (void *) PTRACE_O_TRACESYSGOOD) < 0)
               		die("ptrace(PTRACE_SETOPTIONS): %m");
               	      ptrace(PTRACE_SYSCALL, box_pid, 0, 0);
               	    }
               	  else if (sig != SIGXCPU && sig != SIGXFSZ)
               	    {
               	      msg(">> Signal %d\n", sig);
               	      sample_mem_peak();			/* Signal might be fatal, so update mem-peak */
               	      ptrace(PTRACE_SYSCALL, box_pid, 0, sig);
               	    }
               	  else
               	    {
               	      meta_printf("exitsig:%d", sig);
               	      err("SG: Received signal %d", sig);
               	    }
               	}
                     else
               	die("wait4: unknown status %x, giving up!", stat);
                   }
               }
               static void
               box_inside(int argc, char **argv)
               {
                 struct rlimit rl;
                 char *args[argc+1];
                 memcpy(args, argv, argc * sizeof(char *));
                 args[argc] = NULL;
                 if (set_cwd && chdir(set_cwd))
                   die("chdir: %m");
                 if (redir_stdin)
                   {
                     close(0);
                     if (open(redir_stdin, O_RDONLY) != 0)
               	die("open(\"%s\"): %m", redir_stdin);
                   }
                 if (redir_stdout)
                   {
                     close(1);
                     if (open(redir_stdout, O_WRONLY | O_CREAT | O_TRUNC, 0666) != 1)
               	die("open(\"%s\"): %m", redir_stdout);
                   }
                 if (redir_stderr)
                   {
                     close(2);
                     if (open(redir_stderr, O_WRONLY | O_CREAT | O_TRUNC, 0666) != 2)
               	die("open(\"%s\"): %m", redir_stderr);
                   }
                 else
                   dup2(1, 2);
                 setpgrp();
                 if (memory_limit)
                   {
                     rl.rlim_cur = rl.rlim_max = memory_limit * 1024;
                     if (setrlimit(RLIMIT_AS, &rl) < 0)
               	die("setrlimit(RLIMIT_AS): %m");
                   }
                 rl.rlim_cur = rl.rlim_max = (stack_limit ? (rlim_t)stack_limit * 1024 : RLIM_INFINITY);
                 if (setrlimit(RLIMIT_STACK, &rl) < 0)
                   die("setrlimit(RLIMIT_STACK): %m");
                 rl.rlim_cur = rl.rlim_max = 64;
                 if (setrlimit(RLIMIT_NOFILE, &rl) < 0)
                   die("setrlimit(RLIMIT_NOFILE): %m");
                 char **env = setup_environment();
                 if (filter_syscalls)
                   {
                     if (ptrace(PTRACE_TRACEME) < 0)
               	die("ptrace(PTRACE_TRACEME): %m");
                     /* Trick: Make sure that we are stopped until the boxkeeper wakes up. */
                     raise(SIGSTOP);
                   }
                 execve(args[0], args, env);
                 die("execve(\"%s\"): %m", args[0]);
               }
               static void
               usage(void)
               {
                 fprintf(stderr, "Invalid arguments!\n");
                 printf("\
               Usage: box [<options>] -- <command> <arguments>\n\
               \n\
               Options:\n\
               -a <level>\tSet file access level (0=none, 1=cwd, 2=/etc,/lib,..., 3=whole fs, 9=no checks; needs -f)\n\
               -c <dir>\tChange directory to <dir> first\n\
               -e\t\tInherit full environment of the parent process\n\
               -E <var>\tInherit the environment variable <var> from the parent process\n\
               -E <var>=<val>\tSet the environment variable <var> to <val>; unset it if <var> is empty\n\
               -f\t\tFilter system calls (-ff=very restricted)\n\
               -i <file>\tRedirect stdin from <file>\n\
               -k <size>\tLimit stack size to <size> KB (default: 0=unlimited)\n\
               -m <size>\tLimit address space to <size> KB\n\
               -M <file>\tOutput process information to <file> (name:value)\n\
               -o <file>\tRedirect stdout to <file>\n\
               -p <path>\tPermit access to the specified path (or subtree if it ends with a `/')\n\
               -p <path>=<act>\tDefine action for the specified path (<act>=yes/no)\n\
               -r <file>\tRedirect stderr to <file>\n\
               -s <sys>\tPermit the specified syscall (be careful)\n\
               -s <sys>=<act>\tDefine action for the specified syscall (<act>=yes/no/file)\n\
               -t <time>\tSet run time limit (seconds, fractions allowed)\n\
               -T\t\tAllow syscalls for measuring run time\n\
               -v\t\tBe verbose (use multiple times for even more verbosity)\n\
               -w <time>\tSet wall clock time limit (seconds, fractions allowed)\n\
               -x <time>\tSet extra timeout, before which a timing-out program is not yet killed,\n\
               \t\tso that its real execution time is reported (seconds, fractions allowed)\n\
               ");
                 exit(2);
               }
               int
               main(int argc, char **argv)
               {
                 int c;
                 uid_t uid;
                 while ((c = getopt(argc, argv, "a:c:eE:fi:k:m:M:o:p:r:s:t:Tvw:x:")) >= 0)
                   switch (c)
                     {
                     case 'a':
               	file_access = atol(optarg);
               	break;
                     case 'c':
               	set_cwd = optarg;
               	break;
                     case 'e':
               	pass_environ = 1;
               	break;
                     case 'E':
               	if (!set_env_action(optarg))
               	  usage();
               	break;
                     case 'f':
               	filter_syscalls++;
               	break;
                     case 'k':
               	stack_limit = atol(optarg);
               	break;
                     case 'i':
               	redir_stdin = optarg;
               	break;
                     case 'm':
               	memory_limit = atol(optarg);
               	break;
                     case 'M':
               	meta_open(optarg);
               	break;
                     case 'o':
               	redir_stdout = optarg;
               	break;
                     case 'p':
               	if (!set_path_action(optarg))
               	  usage();
               	break;
                     case 'r':
               	redir_stderr = optarg;
               	break;
                     case 's':
               	if (!set_syscall_action(optarg))
               	  usage();
               	break;
                     case 't':
               	timeout = 1000*atof(optarg);
               	break;
                     case 'T':
               	syscall_action[__NR_times] = A_YES;
               	break;
                     case 'v':
               	verbose++;
               	break;
                     case 'w':
                       wall_timeout = 1000*atof(optarg);
               	break;
                     case 'x':
               	extra_timeout = 1000*atof(optarg);
               	break;
                     default:
               	usage();
                     }
                 if (optind >= argc)
                   usage();
                 sanity_check();
                 uid = geteuid();
                 if (setreuid(uid, uid) < 0)
                   die("setreuid: %m");
                 box_pid = fork();
                 if (box_pid < 0)
                   die("fork: %m");
                 if (!box_pid)
                   box_inside(argc-optind, argv+optind);
                 else
                   boxkeeper();
                 die("Internal error: fell over edge of the world");
               }

std-script/grade ¶ +27 -2

               #!/usr/bin/env ruby
               CORRECT_MARK = 'P'
               INCORRECT_MARK = '-'
               TIMEOUT_MARK = 'T'
               RUN_ERROR_MARK = 'x'
               def log(str='')
                 if ENV['TALKATIVE']!=nil
                   puts str
                 end
                 if ENV['GRADER_LOGGING']!=nil
                   log_fname = ENV['GRADER_LOGGING']
                   fp = File.open(log_fname,"a")
                   fp.puts("grade: #{Time.new.strftime("%H:%M")} #{str}")
                   fp.close
                 end
               end
               def char_comment(comment)
                 if comment =~ /[Ii]ncorrect/
                   INCORRECT_MARK
                 elsif comment =~ /[Cc]orrect/
                   CORRECT_MARK
                 elsif comment =~ /[Tt]ime/
                   TIMEOUT_MARK
                 elsif res = /^[Cc]omment:(.*)$/.match(comment)
                   res[1]
                 else
                   RUN_ERROR_MARK  # these are run time errors
                 end
               end
+            + def extract_time(t)
+            +   puts "TIME: #{t}"
+            +   if (result=/^(.*)r(.*)u(.*)s(.*)m/.match(t))
+            +     {:real => result[1], :user => result[2], :sys => result[3], :mem => result[4]}
+            +   else
+            +     #{:real => 0, :user => 0, :sys => 0}
+            +     #puts "ERROR READING RUNNING TIME: #{t}"
+            +     raise "Error reading running time: #{t}"
+            +   end
+            + end
+            +
               problem_home = ENV['PROBLEM_HOME']
               require "#{problem_home}/script/test_dsl.rb"
               load "#{problem_home}/test_cases/all_tests.cfg"
               problem = Problem.get_instance
               if problem.well_formed? == false
                 log "The problem specification is not well formed."
                 exit(127)
               end
               all_score = 0
               all_comment = ''
+            + peak_memory = -1
+            + max_runtime = -1
               (1..(problem.runs.length-1)).each do |k|
                 log "grade run #{k}"
                 run = problem.runs[k]
                 run_score = nil
                 run_comment = ''
                 run_comment_short = ''
                 run.tests.each do |test_num|
                   result_file_name = "#{test_num}/result"
                   if not File.exists?(result_file_name)
                     run_comment += "result file for test #{test_num} not found\n"
                     run_comment_short += RUN_ERROR_MARK
                     log "Cannot find the file #{test_num}/result!"
                   else
                     result_file = File.new(result_file_name, "r")
                     result_file_lines = result_file.readlines
-            -       if result_file_lines.length>=2
+            +       if result_file_lines.length>=3
                       current_run_score = result_file_lines[1].to_i
                       run_comment += result_file_lines[0]
                       run_comment_short += char_comment(result_file_lines[0].chomp)
+            +
+            +         #update max runtime & memory
+            +         run_stat = extract_time result_file_lines[2]
+            +         peak_memory = [peak_memory,run_stat[:mem].to_i].max
+            +         max_runtime = [max_runtime,run_stat[:user].to_f + run_stat[:sys].to_f].max
                     else
                       current_run_score = 0
                       run_comment += "result file for test #{test_num} error\n"
                       run_comment_short += RUN_ERROR_MARK
                       log "Error in #{test_num}/result!"
                     end
                     # the score of this run should be the minimum of the score for
                     #   each test case
                     if (run_score==nil) or (run_score>current_run_score)
                       run_score = current_run_score
                     end
                     result_file.close
                   end
                 end
                 run_result_file = File.new("result-#{k}", "w")
                 run_result_file.write run_score
                 run_result_file.write "\n"
                 run_result_file.close
                 run_comment_file = File.new("comment-#{k}", "w")
                 run_comment_file.write "#{run_comment}\n"
                 run_comment_file.close
                 all_score = all_score + run_score
                 # append comment for test run with many test cases
                 if run.tests.length > 1
                   run_comment_short = '[' + run_comment_short + ']'
                 end
                 all_comment += run_comment_short
               end
               result_file = File.new("result", "w")
               result_file.write all_score
               result_file.write "\n"
               result_file.close
               comment_file = File.new("comment", "w")
               comment_file.write "#{all_comment}\n"
               comment_file.close
-            - log "score = #{all_score} comment = #{all_comment}"
+            + File.open("run_stat","w") do |file|
+            +   file.puts max_runtime
+            +   file.puts peak_memory
+            + end
+            +
+            + log "score = #{all_score}\ncomment = #{all_comment}"
+            + log "max_runtime = #{max_runtime}\npeak_memory = #{peak_memory}"

Write
Preview

You need to be logged in to leave comments. Login now

	Site-wide shortcuts
/	Use quick search box
g h	Goto home page
g g	Goto my private gists page
g G	Goto my public gists page
n r	New repository page
n g	New gist page

	Repositories
g s	Goto summary page
g c	Goto changelog page
g f	Goto files page
g F	Goto files page with file search activated
g p	Goto pull requests page
g o	Goto repository settings
g O	Goto repository permissions settings

Sign in to your account

Author

r160:f0502be39e3b - Fri, 29 Aug 2014 17:04:23 - 3 files changed: 48 inserted, 10 deleted

Sign in to your account

Commit r160:f0502be39e3b public

Author

r160:f0502be39e3b - Fri, 29 Aug 2014 17:04:23 - 3 files changed: 48 inserted, 10 deleted

Commit `r160:f0502be39e3b` public