cafe-grader-judge-scripts Commit

cafe-grader-judge-scripts

Commit `r51:3618759afb50` public

parent | child

Description:

[grader] added fractional timelimit git-svn-id: http://theory.cpe.ku.ac.th/grader/judge/trunk/scripts@191 6386c4cd-e34a-4fa8-8920-d93eb39b512e

Commit status:

[Not Reviewed]

References:

default

Diff options:

| | | |

Comments:

0 Commit comments 0 Inline Comments

Unresolved TODOs:

There are no unresolved TODOs

Add another comment

r51:3618759afb50 - Thu, 24 Apr 2008 01:23:28 - 6 files changed: 66 inserted, 57 deleted

test/data/test2_05sec.c created 644 ¶ +38

@@ -0,0 +1,38
	1	+ #include <stdio.h>
	2	+ #include <stdlib.h>
	3	+ #include <unistd.h>
	4	+ #include <sys/time.h>
	5	+ #include <time.h>
	6	+ #include <sys/resource.h>
	7	+
	8	+ // run it for 1.5 s
	9	+
	10	+ int main()
	11	+ {
	12	+ int a,b;
	13	+
	14	+ int c=0;
	15	+
	16	+ scanf("%d %d",&a,&b);
	17	+ printf("%d\n",a+b);
	18	+
	19	+ struct rusage ru;
	20	+
	21	+ while(1) {
	22	+ c++;
	23	+ b+=c;
	24	+ while(c<100000) {
	25	+ c++;
	26	+ b+=c;
	27	+ }
	28	+ getrusage(RUSAGE_SELF,&ru);
	29	+ double rtime = ru.ru_utime.tv_sec + ru.ru_stime.tv_sec;
	30	+ rtime += (double)ru.ru_utime.tv_usec / 1000000.0;
	31	+ rtime += (double)ru.ru_stime.tv_usec / 1000000.0;
	32	+ if(rtime > 0.5)
	33	+ break;
	34	+ }
	35	+ printf("%d\n",b);
	36	+ exit(0);
	37	+ }
	38	+

grader ¶ +10 -5

                                                      Process.pid,
                                                      grader_mode)
               else
                 grader_proc = nil
               end
               #set loggin environment
               ENV['GRADER_LOGGING'] = log_file_name
               #
               # MAIN LOOP
               #
               case grader_mode
               when "queue", "test_request"
                 log "Grader: #{grader_mode}"
                 if grader_mode=="queue"
                   engine = Grader::Engine.new
                 else
                   engine = Grader::Engine.new(Grader::TestRequestRoomMaker.new,
                                               Grader::TestRequestReporter.new)
                 end
                 runner = Grader::Runner.new(engine, grader_proc)
                 while true
                   if check_stopfile    # created by calling grader stop
                     clear_stopfile
                     log "stopped (with stop file)"
                     break
                   end
                   if grader_mode=="queue"
                     task = runner.grade_oldest_task
                   else
                     task = runner.grade_oldest_test_request
                   end
                   if task==nil
                     sleep(1)
                   end
                 end
               when "prob"
                 engine = Grader::Engine.new
                 runner = Grader::Runner.new(engine, grader_proc)
                 grader_proc.report_active if grader_proc!=nil
-            -   prob = Problem.find_by_name(ARGV[2])
+            +   ARGV.shift
-            -   if prob==nil
+            +   ARGV.shift
-            -     puts "cannot find problem: #{ARGV[2]}"
-            -   else
+            +   ARGV.each do |prob_name|
-            -     runner.grade_problem(prob)
+            +     prob = Problem.find_by_name(prob_name)
+            +     if prob==nil
+            +       puts "cannot find problem: #{prob_name}"
+            +     else
+            +       runner.grade_problem(prob)
+            +     end
                 end
               else
                 display_manual
                 exit(0)
               end
               # report inactive
               grader_proc.report_inactive if grader_proc!=nil

std-script/box.cc ¶ +10 -7

               /*
                *	A Simple Testing Sandbox
                *
                *	(c) 2001--2004 Martin Mares <mj@ucw.cz>
                */
               #define _LARGEFILE64_SOURCE
               //#define _GNU_SOURCE
               #include <errno.h>
               #include <stdio.h>
               #include <fcntl.h>
               #include <stdlib.h>
               #include <string.h>
               #include <stdarg.h>
               #include <unistd.h>
               #include <getopt.h>
               #include <time.h>
               #include <sys/wait.h>
               #include <sys/user.h>
               #include <sys/time.h>
               #include <sys/ptrace.h>
               #include <sys/signal.h>
               #include <sys/sysinfo.h>
               #include <sys/syscall.h>
               #include <sys/resource.h>
               #define NONRET __attribute__((noreturn))
               #define UNUSED __attribute__((unused))
               static int filter_syscalls;		/* 0=off, 1=liberal, 2=totalitarian */
-            - static int timeout;
+            + static double timeout;
               static int pass_environ;
               static int use_wall_clock;
               static int file_access;
               static int verbose;
               static int memory_limit;
               static int allow_times;
               static char *redir_stdin, *redir_stdout;
               static char *set_cwd;
               static pid_t box_pid;
               static int is_ptraced;
               static volatile int timer_tick;
               static time_t start_time;
               static int ticks_per_sec;
               static int page_size;
               #if defined(__GLIBC__) && __GLIBC__ == 2 && __GLIBC_MINOR__ > 0
               /* glibc 2.1 or newer -> has lseek64 */
               #define long_seek(f,o,w) lseek64(f,o,w)
               #else
               /* Touching clandestine places in glibc */
               extern loff_t llseek(int fd, loff_t pos, int whence);
               #define long_seek(f,o,w) llseek(f,o,w)
               #endif
               int max_mem_used = 0;
               void print_running_stat(double wall_time,
               			double user_time,
               			double system_time,
               			int mem_usage)
               {
                 fprintf(stderr,"%.4lfr%.4lfu%.4lfs%dm\n",
               	  wall_time, user_time, system_time, mem_usage);
               }
               static void NONRET
               box_exit(void)
               {
                 if (box_pid > 0) {
                   if (is_ptraced)
                     ptrace(PTRACE_KILL, box_pid);
                   kill(-box_pid, SIGKILL);
                   kill(box_pid, SIGKILL);
                 }
                 struct timeval total;
                 int wall;
                   case __NR_getdents64:
                   case __NR__newselect:
                   case __NR_fdatasync:
                   case __NR_mremap:
                   case __NR_poll:
                   case __NR_getcwd:
                   case __NR_nanosleep:
                   case __NR_rt_sigreturn:
                   case __NR_rt_sigaction:
                   case __NR_rt_sigprocmask:
                   case __NR_rt_sigpending:
                   case __NR_rt_sigtimedwait:
                   case __NR_rt_sigqueueinfo:
                   case __NR_rt_sigsuspend:
                   case __NR_mmap2:
                   case __NR__sysctl:
                     return (filter_syscalls == 1);
                   case __NR_times:
                     return allow_times;
                   case __NR_kill:
                     if (u->regs.ebx == box_pid)
               	die("Commited suicide by signal %d.", (int)u->regs.ecx);
                     return 0;
                   default:
                     return 0;
                   }
               }
               static void
               signal_alarm(int unused UNUSED)
               {
                 /* Time limit checks are synchronous, so we only schedule them there. */
                 timer_tick = 1;
                 //NOTE: do not use alarm, changed to setitimer for precision
                 //  alarm(1);
               }
               static void
               signal_int(int unused UNUSED)
               {
                 /* Interrupts are fatal, so no synchronization requirements. */
                 die("Interrupted.");
               }
               static void
               check_timeout(void)
               {
-            -   int sec;
+            +   double sec;
                 if (use_wall_clock)
-            -     sec = time(NULL) - start_time;
+            +     sec = (double)(time(NULL) - start_time);
                 else
                   {
                     char buf[4096], *x;
                     int c, utime, stime;
                     static int proc_status_fd;
                     if (!proc_status_fd)
               	{
               	  sprintf(buf, "/proc/%d/stat", (int) box_pid);
               	  proc_status_fd = open(buf, O_RDONLY);
               	  if (proc_status_fd < 0)
               	    die("open(%s): %m", buf);
               	}
                     lseek(proc_status_fd, 0, SEEK_SET);
                     if ((c = read(proc_status_fd, buf, sizeof(buf)-1)) < 0)
               	die("read on /proc/$pid/stat: %m");
                     if (c >= (int) sizeof(buf) - 1)
               	die("/proc/$pid/stat too long");
                     buf[c] = 0;
                     x = buf;
                     while (*x && *x != ' ')
               	x++;
                     while (*x == ' ')
               	x++;
                     if (*x++ != '(')
               	die("proc syntax error 1");
                     while (*x && (*x != ')' || x[1] != ' '))
               	x++;
                     while (*x == ')' || *x == ' ')
               	x++;
                     if (sscanf(x, "%*c %*d %*d %*d %*d %*d %*d %*d %*d %*d %*d %d %d", &utime, &stime) != 2)
               	die("proc syntax error 2");
                     //printf("%s - %d\n",x,ticks_per_sec);
-            -       sec = (utime + stime + ticks_per_sec-1)/ticks_per_sec;
+            +       sec = ((double)(utime + stime))/(double)ticks_per_sec;
                   }
                 if (verbose > 1)
                   fprintf(stderr, "[timecheck: %d seconds]\n", sec);
                 if (sec > timeout) {
-            -     die("Time limit exceeded.");
+            +     die("Time limit exceeded.",sec,timeout);
                 }
               }
               static void
               check_memory_usage()
               {
                 char proc_fname[100];
                 sprintf(proc_fname,"/proc/%d/statm",box_pid);
                 //printf("proc fname: %s\n",proc_fname);
                 FILE *fp = fopen(proc_fname,"r");
                 if(fp!=NULL) {
                   char line[1000];
                   fgets(line,999,fp);
                   //printf("%s\n",line);
                   int m;
                   if(sscanf(line,"%d",&m)==1) {
                     m = (m*page_size+1023)/1024;
                     if(m>max_mem_used)
               	max_mem_used = m;
                   }
                   fclose(fp);
                 }
               }
               static void
               boxkeeper(void)
               {
                 int syscall_count = 0;
                 struct sigaction sa;
                 is_ptraced = 1;
                 bzero(&sa, sizeof(sa));
                 sa.sa_handler = signal_int;
                 sigaction(SIGINT, &sa, NULL);
                 start_time = time(NULL);
                 ticks_per_sec = sysconf(_SC_CLK_TCK);
                 page_size = getpagesize();
                 if (ticks_per_sec <= 0)
                   die("Invalid ticks_per_sec!");
                 check_memory_usage();
                 sa.sa_handler = signal_alarm;
                 sigaction(SIGALRM, &sa, NULL);
                 //alarm(1);
                 val.it_interval.tv_sec = 0;
                 val.it_interval.tv_usec = 50000;
                 val.it_value.tv_sec = 0;
                 val.it_value.tv_usec = 50000;
                 setitimer(ITIMER_REAL,&val,NULL);
                 /*
                   --- add alarm handler no matter what..
                 if (timeout)
                   {
                     sa.sa_handler = signal_alarm;
                     sigaction(SIGALRM, &sa, NULL);
                     alarm(1);
                   }
                 */
                 for(;;)
                   {
                     struct rusage rus;
                     int stat;
                     pid_t p;
                     if (timer_tick)
               	{
               	  check_timeout();
               	  check_memory_usage();
               	  timer_tick = 0;
               	}
                     p = wait4(box_pid, &stat, WUNTRACED, &rus);
                     if (p < 0)
               	{
               	  if (errno == EINTR)
               	    continue;
               	  die("wait4: %m");
               	}
                     if (p != box_pid)
               	die("wait4: unknown pid %d exited!", p);
                     if (WIFEXITED(stat))
               	{
               	  struct timeval total;
               	  int wall;
               	  wall = time(NULL) - start_time;
               	  timeradd(&rus.ru_utime, &rus.ru_stime, &total);
               	  box_pid = 0;
               	  if (WEXITSTATUS(stat))
               	    fprintf(stderr,"Exited with error status %d.\n", WEXITSTATUS(stat));
-            - 	  else if ((use_wall_clock ? wall : total.tv_sec) > timeout)
+            + 	  else if ((use_wall_clock ?
+            + 		    wall :
+            + 		    (double) total.tv_sec +
+            + 		    ((double) total.tv_usec/1000000.0)) > timeout)
               	    fprintf(stderr,"Time limit exceeded.\n");
               	  else
               	    // report OK and statistics
               	    fprintf(stderr,"OK\n");
               	  print_running_stat((double) wall,
               			     (double) rus.ru_utime.tv_sec +
               			     ((double) rus.ru_utime.tv_usec/1000000.0),
               			     (double) rus.ru_stime.tv_sec +
               			     ((double) rus.ru_stime.tv_usec/1000000.0),
               			     max_mem_used);
               /*
               	  (%.4lf sec real (%d), %d sec wall, %d syscalls, %d kb)\n",
               		  (double) total.tv_sec + ((double)total.tv_usec / 1000000.0),
               		  (int) total.tv_usec,
               		  wall,
               		  syscall_count,
               		  max_mem_used);
               */
               	  exit(0);
               	}
                     if (WIFSIGNALED(stat))
               	{
               	  box_pid = 0;
               	  fprintf(stderr,"Caught fatal signal %d.\n", WTERMSIG(stat));
               	  struct timeval total;
               	  int wall;
               	  wall = time(NULL) - start_time;
               	  timeradd(&rus.ru_utime, &rus.ru_stime, &total);
               	  print_running_stat((double) wall,
               			     (double) rus.ru_utime.tv_sec +
               			     ((double) rus.ru_utime.tv_usec/1000000.0),
               			     (double) rus.ru_stime.tv_sec +
               			     ((double) rus.ru_stime.tv_usec/1000000.0),
               			     max_mem_used);
               	  exit(0);
               	}
                     if (WIFSTOPPED(stat))
               	{
               	  int sig = WSTOPSIG(stat);
               	  if (sig == SIGTRAP)
               	    {
               	      struct user u;
               	      static int stop_count = -1;
               	      if (ptrace(PTRACE_GETREGS, box_pid, NULL, &u) < 0)
               		die("ptrace(PTRACE_GETREGS): %m");
               	      stop_count++;
               \n\
               Options:\n\
               -a <level>\tSet file access level (0=none, 1=cwd, 2=/etc,/lib,..., 3=whole fs, 9=no checks; needs -f)\n\
               -c <dir>\tChange directory to <dir> first\n\
               -e\t\tPass full environment of parent process\n\
               -f\t\tFilter system calls (-ff=very restricted)\n\
               -i <file>\tRedirect stdin from <file>\n\
               -m <size>\tLimit address space to <size> KB\n\
               -o <file>\tRedirect stdout to <file>\n\
               -t <time>\tStop after <time> seconds\n\
               -T\t\tAllow syscalls for measuring run time\n\
               -v\t\tBe verbose\n\
               -w\t\tMeasure wall clock time instead of run time\n\
               ");
                 exit(1);
               }
               int
               main(int argc, char **argv)
               {
                 int c;
                 uid_t uid;
                 while ((c = getopt(argc, argv, "a:c:efi:m:o:t:Tvw")) >= 0)
                   switch (c)
                     {
                     case 'a':
               	file_access = atol(optarg);
               	break;
                     case 'c':
               	set_cwd = optarg;
               	break;
                     case 'e':
               	pass_environ = 1;
               	break;
                     case 'f':
               	filter_syscalls++;
               	break;
                     case 'i':
               	redir_stdin = optarg;
               	break;
                     case 'm':
               	memory_limit = atol(optarg);
               	break;
                     case 'o':
               	redir_stdout = optarg;
               	break;
                     case 't':
-            - 	timeout = atol(optarg);
+            + 	timeout = atof(optarg);
               	break;
                     case 'T':
               	allow_times++;
               	break;
                     case 'v':
               	verbose++;
               	break;
                     case 'w':
               	use_wall_clock = 1;
               	break;
                     default:
               	usage();
                     }
                 if (optind >= argc)
                   usage();
                 uid = geteuid();
                 if (setreuid(uid, uid) < 0)
                   die("setreuid: %m");
                 box_pid = fork();
                 if (box_pid < 0)
                   die("fork: %m");
                 if (!box_pid)
                   box_inside(argc-optind, argv+optind);
                 else
                   boxkeeper();
                 die("Internal error: fell over edge of the world");
               }

test/data/ev/test_timeout/test_cases/all_tests.cfg ¶ +5 -1

               problem do
                 num_tests 2
                 full_score 20
                 time_limit_each 1
                 mem_limit_each 16
                 score_each 10
+            +   test 1 do
+            +     time_limit 0.5
+            +   end
+            +
                 run 1 do
                   tests 1
                 end
                 test 2 do
-            -     time_limit 2
+            +     time_limit 0.6
                 end
                 run 2 do
                   tests 2
                 end
               end

test/engine_spec.rb ¶ +3 -3

               describe "A grader engine, when grading submissions" do
                 include GraderEngineHelperMethods
                 before(:each) do
                   @config = Grader::Configuration.get_instance
                   # this test is from Pong
                   @problem_test_normal = stub(Problem,
                                               :id => 1, :name => 'test_normal',
                                               :full_score => 135)
                   @user_user1 = stub(User,
                                      :id => 1, :login => 'user1')
                   @engine = Grader::Engine.new
                   init_sandbox
                 end
                 it "should grade normal submission" do
                   grader_should(:grade => "test1_correct.c",
                                 :on => @problem_test_normal,
                                 :and_report => {
                                   :score => 135,
                                   :comment => /^PASSED/})
                 end
                 it "should produce error message when submission cannot compile" do
                   grader_should(:grade => "test1_compile_error.c",
                                 :on => @problem_test_normal,
                                 :and_report => {
                                   :score => 0,
                                   :comment => 'FAILED: compilation error',
                                   :compiler_message => /[Ee]rror/})
                 end
                 it "should produce timeout error when submission runs forever" do
                   @problem_test_timeout = stub(Problem,
                                                :id => 1, :name => 'test_timeout',
                                                :full_score => 10)
                   grader_should(:grade => "test2_timeout.c",
                                 :on => @problem_test_timeout,
                                 :and_report => {
                                   :score => 0,
                                   :comment => 'FAILED: TT'})
                 end
-            -   it "should produce timeout error correctly when submission runs slower than expected in less than a second" do
+            +   it "should produce timeout error correctly with fractional running time and fractional time limits" do
-            -     @problem_test_timeout = stub(Problem,
+            +      @problem_test_timeout = stub(Problem,
                                                :id => 1, :name => 'test_timeout',
                                                :full_score => 20)
-            -     grader_should(:grade => "test2_1-5sec.c",
+            +     grader_should(:grade => "test2_05sec.c",
                                 :on => @problem_test_timeout,
                                 :and_report => {
                                   :score => 10,
                                   :comment => 'FAILED: TP'})
                 end
                 it "should produce runtime error when submission uses too much static memory" do
                   @problem_test_memory = stub(Problem,
                                               :id => 1, :name => 'test_memory',
                                               :full_score => 20)
                   grader_should(:grade => "add_too_much_memory_static.c",
                                 :on => @problem_test_memory,
                                 :and_report => {
                                   :score => 10,
                                   :comment => /FAILED: [^P]P/})
                 end
                 it "should not allow submission to allocate too much dynamic memory" do
                   @problem_test_memory = stub(Problem,
                                               :id => 1, :name => 'test_memory',
                                               :full_score => 20)
                   grader_should(:grade => "add_too_much_memory_dynamic.c",
                                 :on => @problem_test_memory,
                                 :and_report => {
                                   :score => 10,
                                   :comment => /FAILED: [^P]P/})
                 end
                 it "should score test runs correctly when submission fails in some test case" do
                   grader_should(:grade => "add_fail_test_case_1.c",
                                 :on => @problem_test_normal,
                                 :and_report => {
                                   :score => 105,
                                   :comment => /^FAILED:/})
                 end
                 it "should fail submission with non-zero exit status" do
                   grader_should(:grade => "add_nonzero_exit_status.c",
                                 :on => @problem_test_normal,
                                 :and_report => {
                                   :score => 0,
                                   :comment => /^FAILED:/})
                 end
                 it "should not allow malicious submission to see PROBLEM_HOME" do
                   problem_test_yesno =  stub(Problem,
                                               :id => 1, :name => 'test_yesno',
                                               :full_score => 10)

test/data/test2_1-5sec.c removed ¶ -41

deleted file

Write
Preview

You need to be logged in to leave comments. Login now

	Site-wide shortcuts
/	Use quick search box
g h	Goto home page
g g	Goto my private gists page
g G	Goto my public gists page
n r	New repository page
n g	New gist page

	Repositories
g s	Goto summary page
g c	Goto changelog page
g f	Goto files page
g F	Goto files page with file search activated
g p	Goto pull requests page
g o	Goto repository settings
g O	Goto repository permissions settings

Sign in to your account

Author

r51:3618759afb50 - Thu, 24 Apr 2008 01:23:28 - 6 files changed: 66 inserted, 57 deleted

Sign in to your account

Commit r51:3618759afb50 public

Author

r51:3618759afb50 - Thu, 24 Apr 2008 01:23:28 - 6 files changed: 66 inserted, 57 deleted

Commit `r51:3618759afb50` public