cafe-grader-judge-scripts

Commit r46:31f3935d33aa public

parent | child
Description:
[grader] box.cc now reports running status when die git-svn-id: http://theory.cpe.ku.ac.th/grader/judge/trunk/scripts@172 6386c4cd-e34a-4fa8-8920-d93eb39b512e
Commit status:
[Not Reviewed]
References:
default
Diff options:
Raw Diff | Patch Diff | Download Diff | Ignore whitespace | Increase context
Comments:
0 Commit comments 0 Inline Comments
Unresolved TODOs:
There are no unresolved TODOs
Side by Side Unified
Expand All Files Collapse All Files Wide Mode Diff
Add another comment
Browse Files

r46:31f3935d33aa - - 2 files changed: 61 inserted, 47 deleted

Show file before | Show file after | Raw diff | Download diff | Ignore whitespace Increase context Show commentsHide comments
@@ -1,197 +1,215
1 1 /*
2 2 * A Simple Testing Sandbox
3 3 *
4 4 * (c) 2001--2004 Martin Mares <mj@ucw.cz>
5 5 */
6 6
7 7 #define _LARGEFILE64_SOURCE
8 8 //#define _GNU_SOURCE
9 9
10 10 #include <errno.h>
11 11 #include <stdio.h>
12 12 #include <fcntl.h>
13 13 #include <stdlib.h>
14 14 #include <string.h>
15 15 #include <stdarg.h>
16 16 #include <unistd.h>
17 17 #include <getopt.h>
18 18 #include <time.h>
19 19 #include <sys/wait.h>
20 20 #include <sys/user.h>
21 21 #include <sys/time.h>
22 22 #include <sys/ptrace.h>
23 23 #include <sys/signal.h>
24 24 #include <sys/sysinfo.h>
25 25 #include <sys/syscall.h>
26 26 #include <sys/resource.h>
27 27
28 28 #define NONRET __attribute__((noreturn))
29 29 #define UNUSED __attribute__((unused))
30 30
31 31 static int filter_syscalls; /* 0=off, 1=liberal, 2=totalitarian */
32 32 static int timeout;
33 33 static int pass_environ;
34 34 static int use_wall_clock;
35 35 static int file_access;
36 36 static int verbose;
37 37 static int memory_limit;
38 38 static int allow_times;
39 39 static char *redir_stdin, *redir_stdout;
40 40 static char *set_cwd;
41 41
42 42 static pid_t box_pid;
43 43 static int is_ptraced;
44 44 static volatile int timer_tick;
45 45 static time_t start_time;
46 46 static int ticks_per_sec;
47 47
48 48 #if defined(__GLIBC__) && __GLIBC__ == 2 && __GLIBC_MINOR__ > 0
49 49 /* glibc 2.1 or newer -> has lseek64 */
50 50 #define long_seek(f,o,w) lseek64(f,o,w)
51 51 #else
52 52 /* Touching clandestine places in glibc */
53 53 extern loff_t llseek(int fd, loff_t pos, int whence);
54 54 #define long_seek(f,o,w) llseek(f,o,w)
55 55 #endif
56 56
57 + int max_mem_used = 0;
58 +
59 + void print_running_stat(double wall_time,
60 + double user_time,
61 + double system_time,
62 + int mem_usage)
63 + {
64 + fprintf(stderr,"%.4lfr%.4lfu%.4lfs%dm\n",
65 + wall_time, user_time, system_time, mem_usage);
66 + }
67 +
57 68 static void NONRET
58 69 box_exit(void)
59 70 {
60 - if (box_pid > 0)
61 - {
62 - if (is_ptraced)
63 - ptrace(PTRACE_KILL, box_pid);
64 - kill(-box_pid, SIGKILL);
65 - kill(box_pid, SIGKILL);
66 - }
67 - exit(1);
68 - }
71 + if (box_pid > 0) {
72 + if (is_ptraced)
73 + ptrace(PTRACE_KILL, box_pid);
74 + kill(-box_pid, SIGKILL);
75 + kill(box_pid, SIGKILL);
76 + }
69 77
70 - static void
71 - box_kill(void)
72 - {
73 - if (box_pid > 0)
74 - {
75 - if (is_ptraced)
76 - ptrace(PTRACE_KILL, box_pid);
77 - kill(-box_pid, SIGKILL);
78 - kill(box_pid, SIGKILL);
79 - }
78 + struct timeval total;
79 + int wall;
80 + struct rusage rus;
81 + int stat;
82 + pid_t p;
83 +
84 + // wait so that we can get information
85 + p = wait4(box_pid, &stat, WUNTRACED, &rus);
86 + if (p < 0) {
87 + fprintf(stderr,"wait4: error\n");
88 + print_running_stat(0,0,0,max_mem_used);
89 + } else if (p != box_pid) {
90 + fprintf(stderr,"wait4: unknown pid %d exited!\n", p);
91 + print_running_stat(0,0,0,max_mem_used);
92 + } else {
93 + if (!WIFEXITED(stat))
94 + fprintf(stderr,"wait4: unknown status\n");
95 + struct timeval total;
96 + int wall;
97 + wall = time(NULL) - start_time;
98 + timeradd(&rus.ru_utime, &rus.ru_stime, &total);
99 +
100 + print_running_stat((double)wall,
101 + (double) rus.ru_utime.tv_sec +
102 + ((double) rus.ru_utime.tv_usec/1000000.0),
103 + (double) rus.ru_stime.tv_sec +
104 + ((double) rus.ru_stime.tv_usec/1000000.0),
105 + max_mem_used);
106 + }
107 + exit(1);
80 108 }
81 109
82 110 static void NONRET __attribute__((format(printf,1,2)))
83 111 die(char *msg, ...)
84 112 {
85 113 va_list args;
86 114 va_start(args, msg);
87 115 vfprintf(stderr, msg, args);
88 116 fputc('\n', stderr);
89 117 box_exit();
90 118 }
91 119
92 - static void __attribute__((format(printf,1,2)))
93 - die_report(char *msg, ...)
94 - {
95 - va_list args;
96 - va_start(args, msg);
97 - vfprintf(stderr, msg, args);
98 - fputc('\n', stderr);
99 - box_kill();
100 - }
101 -
102 120 static void __attribute__((format(printf,1,2)))
103 121 log(char *msg, ...)
104 122 {
105 123 va_list args;
106 124 va_start(args, msg);
107 125 if (verbose)
108 126 {
109 127 vfprintf(stderr, msg, args);
110 128 fflush(stderr);
111 129 }
112 130 va_end(args);
113 131 }
114 132
115 133 static void
116 134 valid_filename(unsigned long addr)
117 135 {
118 136 char namebuf[4096], *p, *end;
119 137 static int mem_fd;
120 138
121 139 if (!file_access)
122 140 die("File access forbidden.");
123 141 if (file_access >= 9)
124 142 return;
125 143
126 144 if (!mem_fd)
127 145 {
128 146 sprintf(namebuf, "/proc/%d/mem", (int) box_pid);
129 147 mem_fd = open(namebuf, O_RDONLY);
130 148 if (mem_fd < 0)
131 149 die("open(%s): %m", namebuf);
132 150 }
133 151 p = end = namebuf;
134 152 do
135 153 {
136 154 if (p >= end)
137 155 {
138 156 int remains = PAGE_SIZE - (addr & (PAGE_SIZE-1));
139 157 int l = namebuf + sizeof(namebuf) - end;
140 158 if (l > remains)
141 159 l = remains;
142 160 if (!l)
143 161 die("Access to file with name too long.");
144 162 if (long_seek(mem_fd, addr, SEEK_SET) < 0)
145 163 die("long_seek(mem): %m");
146 164 remains = read(mem_fd, end, l);
147 165 if (remains < 0)
148 166 die("read(mem): %m");
149 167 if (!remains)
150 168 die("Access to file with name out of memory.");
151 169 end += l;
152 170 addr += l;
153 171 }
154 172 }
155 173 while (*p++);
156 174
157 175 log("[%s] ", namebuf);
158 176 if (file_access >= 3)
159 177 return;
160 178 if (!strchr(namebuf, '/') && strcmp(namebuf, ".."))
161 179 return;
162 180 if (file_access >= 2)
163 181 {
164 182 if ((!strncmp(namebuf, "/etc/", 5) ||
165 183 !strncmp(namebuf, "/lib/", 5) ||
166 184 !strncmp(namebuf, "/usr/lib/", 9))
167 185 && !strstr(namebuf, ".."))
168 186 return;
169 187 if (!strcmp(namebuf, "/dev/null") ||
170 188 !strcmp(namebuf, "/dev/zero") ||
171 189 !strcmp(namebuf, "/proc/meminfo") ||
172 190 !strcmp(namebuf, "/proc/self/stat") ||
173 191 !strncmp(namebuf, "/usr/share/zoneinfo/", 20))
174 192 return;
175 193 }
176 194 die("Forbidden access to file `%s'.", namebuf);
177 195 }
178 196
179 197 static int
180 198 valid_syscall(struct user *u)
181 199 {
182 200 switch (u->regs.orig_eax)
183 201 {
184 202 case __NR_execve:
185 203 {
186 204 static int exec_counter;
187 205 return !exec_counter++;
188 206 }
189 207 case __NR_open:
190 208 case __NR_creat:
191 209 case __NR_unlink:
192 210 case __NR_oldstat:
193 211 case __NR_access:
194 212 case __NR_oldlstat:
195 213 case __NR_truncate:
196 214 case __NR_stat:
197 215 case __NR_lstat:
@@ -256,328 +274,324
256 274 case __NR_readdir:
257 275 case __NR_setitimer:
258 276 case __NR_getitimer:
259 277 case __NR_sigreturn:
260 278 case __NR_mprotect:
261 279 case __NR_sigprocmask:
262 280 case __NR_getdents:
263 281 case __NR_getdents64:
264 282 case __NR__newselect:
265 283 case __NR_fdatasync:
266 284 case __NR_mremap:
267 285 case __NR_poll:
268 286 case __NR_getcwd:
269 287 case __NR_nanosleep:
270 288 case __NR_rt_sigreturn:
271 289 case __NR_rt_sigaction:
272 290 case __NR_rt_sigprocmask:
273 291 case __NR_rt_sigpending:
274 292 case __NR_rt_sigtimedwait:
275 293 case __NR_rt_sigqueueinfo:
276 294 case __NR_rt_sigsuspend:
277 295 case __NR_mmap2:
278 296 case __NR__sysctl:
279 297 return (filter_syscalls == 1);
280 298 case __NR_times:
281 299 return allow_times;
282 300 case __NR_kill:
283 301 if (u->regs.ebx == box_pid)
284 302 die("Commited suicide by signal %d.", (int)u->regs.ecx);
285 303 return 0;
286 304 default:
287 305 return 0;
288 306 }
289 307 }
290 308
291 309 static void
292 310 signal_alarm(int unused UNUSED)
293 311 {
294 312 /* Time limit checks are synchronous, so we only schedule them there. */
295 313 timer_tick = 1;
296 314
297 315 //NOTE: do not use alarm, changed to setitimer for precision
298 316 // alarm(1);
299 317 }
300 318
301 319 static void
302 320 signal_int(int unused UNUSED)
303 321 {
304 322 /* Interrupts are fatal, so no synchronization requirements. */
305 323 die("Interrupted.");
306 324 }
307 325
308 326 static void
309 327 check_timeout(void)
310 328 {
311 329 int sec;
312 330
313 331 if (use_wall_clock)
314 332 sec = time(NULL) - start_time;
315 333 else
316 334 {
317 335 char buf[4096], *x;
318 336 int c, utime, stime;
319 337 static int proc_status_fd;
320 338 if (!proc_status_fd)
321 339 {
322 340 sprintf(buf, "/proc/%d/stat", (int) box_pid);
323 341 proc_status_fd = open(buf, O_RDONLY);
324 342 if (proc_status_fd < 0)
325 343 die("open(%s): %m", buf);
326 344 }
327 345 lseek(proc_status_fd, 0, SEEK_SET);
328 346 if ((c = read(proc_status_fd, buf, sizeof(buf)-1)) < 0)
329 347 die("read on /proc/$pid/stat: %m");
330 348 if (c >= (int) sizeof(buf) - 1)
331 349 die("/proc/$pid/stat too long");
332 350 buf[c] = 0;
333 351 x = buf;
334 352 while (*x && *x != ' ')
335 353 x++;
336 354 while (*x == ' ')
337 355 x++;
338 356 if (*x++ != '(')
339 357 die("proc syntax error 1");
340 358 while (*x && (*x != ')' || x[1] != ' '))
341 359 x++;
342 360 while (*x == ')' || *x == ' ')
343 361 x++;
344 362 if (sscanf(x, "%*c %*d %*d %*d %*d %*d %*d %*d %*d %*d %*d %d %d", &utime, &stime) != 2)
345 363 die("proc syntax error 2");
346 364 //printf("%s - %d\n",x,ticks_per_sec);
347 365 sec = (utime + stime + ticks_per_sec-1)/ticks_per_sec;
348 366 }
349 367 if (verbose > 1)
350 368 fprintf(stderr, "[timecheck: %d seconds]\n", sec);
351 369 if (sec > timeout) {
352 - die_report("Time limit exceeded.");
370 + die("Time limit exceeded.");
353 371 }
354 372 }
355 373
356 - int max_mem_used = 0;
357 -
358 374 static void
359 375 check_memory_usage()
360 376 {
361 377 char proc_fname[100];
362 378 sprintf(proc_fname,"/proc/%d/statm",box_pid);
363 379 //printf("proc fname: %s\n",proc_fname);
364 380 FILE *fp = fopen(proc_fname,"r");
365 381 if(fp!=NULL) {
366 382 char line[1000];
367 383 fgets(line,999,fp);
368 384 //printf("%s\n",line);
369 385 int m;
370 386
371 387 if(sscanf(line,"%d",&m)==1)
372 388 if(m>max_mem_used)
373 389 max_mem_used = m;
374 390
375 391 fclose(fp);
376 392 }
377 393 }
378 394
379 395 static void
380 396 boxkeeper(void)
381 397 {
382 398 int syscall_count = 0;
383 399 struct sigaction sa;
384 400
385 401 is_ptraced = 1;
386 402 bzero(&sa, sizeof(sa));
387 403 sa.sa_handler = signal_int;
388 404 sigaction(SIGINT, &sa, NULL);
389 405 start_time = time(NULL);
390 406 ticks_per_sec = sysconf(_SC_CLK_TCK);
391 407 if (ticks_per_sec <= 0)
392 408 die("Invalid ticks_per_sec!");
393 409
394 410 check_memory_usage();
395 411
396 412 sa.sa_handler = signal_alarm;
397 413 sigaction(SIGALRM, &sa, NULL);
398 414 //alarm(1);
399 415
400 416 struct itimerval val;
401 417 val.it_interval.tv_sec = 0;
402 418 val.it_interval.tv_usec = 50000;
403 419 val.it_value.tv_sec = 0;
404 420 val.it_value.tv_usec = 50000;
405 421 setitimer(ITIMER_REAL,&val,NULL);
406 422
407 423 /*
408 424 --- add alarm handler no matter what..
409 425 if (timeout)
410 426 {
411 427 sa.sa_handler = signal_alarm;
412 428 sigaction(SIGALRM, &sa, NULL);
413 429 alarm(1);
414 430 }
415 431 */
416 432
417 433 for(;;)
418 434 {
419 435 struct rusage rus;
420 436 int stat;
421 437 pid_t p;
422 438
423 439 if (timer_tick)
424 440 {
425 441 check_timeout();
426 442 check_memory_usage();
427 443 timer_tick = 0;
428 444 }
429 445 p = wait4(box_pid, &stat, WUNTRACED, &rus);
430 446
431 447 if (p < 0)
432 448 {
433 449 if (errno == EINTR)
434 450 continue;
435 451 die("wait4: %m");
436 452 }
437 453 if (p != box_pid)
438 454 die("wait4: unknown pid %d exited!", p);
439 455 if (WIFEXITED(stat))
440 456 {
441 457 struct timeval total;
442 458 int wall;
443 459 wall = time(NULL) - start_time;
444 460 timeradd(&rus.ru_utime, &rus.ru_stime, &total);
445 461
446 462 box_pid = 0;
447 463 if (WEXITSTATUS(stat))
448 464 fprintf(stderr,"Exited with error status %d.\n", WEXITSTATUS(stat));
449 465 else if ((use_wall_clock ? wall : total.tv_sec) > timeout)
450 466 fprintf(stderr,"Time limit exceeded.\n");
451 467 else
452 468 // report OK and statistics
453 469 fprintf(stderr,"OK\n");
454 470
455 - fprintf(stderr,"%dr%.4lfu%.4lfs%dm\n",
456 - wall,
457 - (double) rus.ru_utime.tv_sec +
458 - ((double) rus.ru_utime.tv_usec/1000000.0),
459 - (double) rus.ru_stime.tv_sec +
460 - ((double) rus.ru_stime.tv_usec/1000000.0),
461 - max_mem_used);
471 + print_running_stat((double) wall,
472 + (double) rus.ru_utime.tv_sec +
473 + ((double) rus.ru_utime.tv_usec/1000000.0),
474 + (double) rus.ru_stime.tv_sec +
475 + ((double) rus.ru_stime.tv_usec/1000000.0),
476 + max_mem_used);
462 477 /*
463 478 (%.4lf sec real (%d), %d sec wall, %d syscalls, %d kb)\n",
464 479 (double) total.tv_sec + ((double)total.tv_usec / 1000000.0),
465 480 (int) total.tv_usec,
466 481 wall,
467 482 syscall_count,
468 483 max_mem_used);
469 484 */
470 485 exit(0);
471 486 }
472 487 if (WIFSIGNALED(stat))
473 488 {
474 489 box_pid = 0;
475 490 fprintf(stderr,"Caught fatal signal %d.\n", WTERMSIG(stat));
476 491
477 492 struct timeval total;
478 493 int wall;
479 494 wall = time(NULL) - start_time;
480 495 timeradd(&rus.ru_utime, &rus.ru_stime, &total);
481 - fprintf(stderr,"%dr%.4lfu%.4lfs%dm\n",
482 - wall,
483 - (double) rus.ru_utime.tv_sec +
484 - ((double) rus.ru_utime.tv_usec/1000000.0),
485 - (double) rus.ru_stime.tv_sec +
486 - ((double) rus.ru_stime.tv_usec/1000000.0),
487 - max_mem_used);
496 + print_running_stat((double) wall,
497 + (double) rus.ru_utime.tv_sec +
498 + ((double) rus.ru_utime.tv_usec/1000000.0),
499 + (double) rus.ru_stime.tv_sec +
500 + ((double) rus.ru_stime.tv_usec/1000000.0),
501 + max_mem_used);
488 502 exit(0);
489 503 }
490 504 if (WIFSTOPPED(stat))
491 505 {
492 506 int sig = WSTOPSIG(stat);
493 507 if (sig == SIGTRAP)
494 508 {
495 509 struct user u;
496 510 static int stop_count = -1;
497 511 if (ptrace(PTRACE_GETREGS, box_pid, NULL, &u) < 0)
498 512 die("ptrace(PTRACE_GETREGS): %m");
499 513 stop_count++;
500 514 if (!stop_count) /* Traceme request */
501 515 log(">> Traceme request caught\n");
502 516 else if (stop_count & 1) /* Syscall entry */
503 517 {
504 518 log(">> Syscall %3ld (%08lx,%08lx,%08lx) ", u.regs.orig_eax, u.regs.ebx, u.regs.ecx, u.regs.edx);
505 519 syscall_count++;
506 520 if (!valid_syscall(&u))
507 521 {
508 522 /*
509 523 * Unfortunately, PTRACE_KILL kills _after_ the syscall completes,
510 524 * so we have to change it to something harmless (e.g., an undefined
511 525 * syscall) and make the program continue.
512 526 */
513 527 unsigned int sys = u.regs.orig_eax;
514 528 u.regs.orig_eax = 0xffffffff;
515 529 if (ptrace(PTRACE_SETREGS, box_pid, NULL, &u) < 0)
516 530 die("ptrace(PTRACE_SETREGS): %m");
517 531 die("Forbidden syscall %d.", sys);
518 532 }
519 533 }
520 534 else /* Syscall return */
521 535 log("= %ld\n", u.regs.eax);
522 536 ptrace(PTRACE_SYSCALL, box_pid, 0, 0);
523 537 }
524 538 else if (sig != SIGSTOP && sig != SIGXCPU && sig != SIGXFSZ)
525 539 {
526 540 log(">> Signal %d\n", sig);
527 541 ptrace(PTRACE_SYSCALL, box_pid, 0, sig);
528 542 }
529 543 else
530 544 die("Received signal %d.", sig);
531 545 }
532 546 else
533 547 die("wait4: unknown status %x, giving up!", stat);
534 548 }
535 549 }
536 550
537 551 static void
538 552 box_inside(int argc, char **argv)
539 553 {
540 554 struct rlimit rl;
541 555 char *args[argc+1];
542 556 char *env[1] = { NULL };
543 557
544 558 memcpy(args, argv, argc * sizeof(char *));
545 559 args[argc] = NULL;
546 560 if (set_cwd && chdir(set_cwd))
547 561 die("chdir: %m");
548 562 if (redir_stdin)
549 563 {
550 564 close(0);
551 565 if (open(redir_stdin, O_RDONLY) != 0)
552 566 die("open(\"%s\"): %m", redir_stdin);
553 567 }
554 568 if (redir_stdout)
555 569 {
556 570 close(1);
557 571 if (open(redir_stdout, O_WRONLY | O_CREAT | O_TRUNC, 0666) != 1)
558 572 die("open(\"%s\"): %m", redir_stdout);
559 573 }
560 574 dup2(1, 2);
561 575 setpgrp();
562 576 if (memory_limit)
563 577 {
564 578 rl.rlim_cur = rl.rlim_max = memory_limit * 1024;
565 579 if (setrlimit(RLIMIT_AS, &rl) < 0)
566 580 die("setrlimit: %m");
567 581 }
568 582 rl.rlim_cur = rl.rlim_max = 64;
569 583 if (setrlimit(RLIMIT_NOFILE, &rl) < 0)
570 584 die("setrlimit: %m");
571 585 if (filter_syscalls && ptrace(PTRACE_TRACEME) < 0)
572 586 die("ptrace(PTRACE_TRACEME): %m");
573 587 execve(args[0], args, (pass_environ ? environ : env));
574 588 die("execve(\"%s\"): %m", args[0]);
575 589 }
576 590
577 591 static void
578 592 usage(void)
579 593 {
580 594 fprintf(stderr, "Invalid arguments!\n");
581 595 printf("\
582 596 Usage: box [<options>] -- <command> <arguments>\n\
583 597 \n\
Show file before | Show file after | Raw diff | Download diff | Ignore whitespace Increase context Show commentsHide comments
@@ -148,169 +148,169
148 148 create_submission_from_file(1, @user_user1, @problem_test_normal, source_fname)
149 149 end
150 150
151 151 end
152 152
153 153 describe "A grader engine, when grading test requests" do
154 154
155 155 include GraderEngineHelperMethods
156 156
157 157 before(:each) do
158 158 @config = Grader::Configuration.get_instance
159 159 @engine = Grader::Engine.new(Grader::TestRequestRoomMaker.new,
160 160 Grader::TestRequestReporter.new)
161 161 init_sandbox
162 162 end
163 163
164 164 it "should report error if there is no problem template" do
165 165 problem = stub(Problem,
166 166 :id => 1, :name => 'nothing')
167 167 grader_should(:grade => 'test1_correct.c',
168 168 :on => problem,
169 169 :with => 'in1.txt',
170 170 :and_report => {
171 171 :graded_at= => nil,
172 172 :compiler_message= => '',
173 173 :grader_comment= => '',
174 174 :running_stat= => /template not found/,
175 175 :running_time= => nil,
176 176 :exit_status= => nil,
177 177 :memory_usage= => nil,
178 178 :save => nil})
179 179 end
180 180
181 181 it "should run test request and produce output file" do
182 182 problem = stub(Problem,
183 183 :id => 1, :name => 'test_normal')
184 184 grader_should(:grade => 'test1_correct.c',
185 185 :on => problem,
186 186 :with => 'in1.txt',
187 187 :and_report => {
188 188 :graded_at= => nil,
189 189 :compiler_message= => '',
190 190 :grader_comment= => '',
191 191 :running_stat= => /0.0\d* sec./,
192 192 :output_file_name= => lambda { |fname|
193 193 File.exists?(fname).should be_true
194 194 },
195 195 :running_time= => nil,
196 196 :exit_status= => nil,
197 197 :memory_usage= => nil,
198 198 :save => nil})
199 199 end
200 200
201 201 it "should clean up problem directory after running test request" do
202 202 problem = stub(Problem,
203 203 :id => 1, :name => 'test_normal')
204 204 grader_should(:grade => 'test1_correct.c',
205 205 :on => problem,
206 206 :with => 'in1.txt',
207 207 :and_report => {
208 208 :graded_at= => nil,
209 209 :compiler_message= => '',
210 210 :grader_comment= => '',
211 211 :running_stat= => nil,
212 212 :output_file_name= => nil,
213 213 :running_time= => nil,
214 214 :exit_status= => nil,
215 215 :memory_usage= => nil,
216 216 :save => nil})
217 217 File.exists?(@config.user_result_dir + "/test_request/test_normal/test_cases/1/input-1.txt").should be_false
218 218 end
219 219
220 220 it "should compile test request with error and report compilation error" do
221 221 problem = stub(Problem,
222 222 :id => 1, :name => 'test_normal')
223 223 grader_should(:grade => 'test1_compile_error.c',
224 224 :on => problem,
225 225 :with => 'in1.txt',
226 226 :and_report => {
227 227 :graded_at= => nil,
228 228 :compiler_message= => /.+/,
229 229 :grader_comment= => /[Cc]ompil.*error/,
230 230 :running_stat= => '',
231 231 :save => nil})
232 232 end
233 233
234 234 it "should report exit status" do
235 235 problem = stub(Problem,
236 236 :id => 1, :name => 'test_normal')
237 237 grader_should(:grade => 'add_nonzero_exit_status.c',
238 238 :on => problem,
239 239 :with => 'in1.txt',
240 240 :and_report => {
241 241 :graded_at= => nil,
242 242 :compiler_message= => '',
243 243 :grader_comment= => '',
244 - :running_stat= => /[Ee]xit.*status.*10.*0\.0 sec/m,
244 + :running_stat= => /[Ee]xit.*status.*10.*0\.0\d* sec/m,
245 245 :output_file_name= => lambda { |fname|
246 246 File.exists?(fname).should be_true
247 247 },
248 248 :running_time= => nil,
249 249 :exit_status= => /10/,
250 250 :memory_usage= => nil,
251 251 :save => nil})
252 252 end
253 253
254 254 it "should produce running statistics for normal submission" do
255 255 problem = stub(Problem,
256 256 :id => 1, :name => 'test_normal')
257 257 grader_should(:grade => 'test_run_stat.c',
258 258 :on => problem,
259 259 :with => 'in1.txt',
260 260 :and_report => {
261 261 :graded_at= => nil,
262 262 :compiler_message= => '',
263 263 :grader_comment= => '',
264 264 :running_stat= => nil,
265 265 :output_file_name= => lambda { |fname|
266 266 File.exists?(fname).should be_true
267 267 },
268 268 :running_time= => lambda { |t|
269 269 (t>=0.14) and (t<=0.16)
270 270 },
271 271 :exit_status= => nil,
272 272 :memory_usage= => lambda { |s|
273 273 (s>=500) and (s<=1000)
274 274 },
275 275 :save => nil})
276 276 end
277 277
278 278 protected
279 279 def grader_should(args)
280 280 @user1 = stub(User,
281 281 :id => 1, :login => 'user1')
282 282
283 283 problem = args[:on]
284 284 input_file = @config.test_request_input_base_dir + "/" + args[:with]
285 285
286 286 submission =
287 287 create_submission_from_file(1, @user1, args[:on], args[:grade])
288 288
289 289 test_request = stub(TestRequest,
290 290 :id => 1,
291 291 :user => @user1,
292 292 :problem => problem,
293 293 :submission => submission,
294 294 :input_file_name => input_file,
295 295 :language => submission.language,
296 296 :problem_name => problem.name)
297 297
298 298 expectations = args[:and_report]
299 299
300 300 expectations.each do |key,val|
301 301 if val==nil
302 302 test_request.should_receive(key)
303 303 elsif val.class == Proc
304 304 test_request.should_receive(key) { |fname|
305 305 val.call(fname)
306 306 }
307 307 else
308 308 test_request.should_receive(key).with(val)
309 309 end
310 310 end
311 311
312 312 @engine.grade(test_request)
313 313 end
314 314
315 315 end
316 316
You need to be logged in to leave comments. Login now