cafe-grader-judge-scripts

Commit r210:3ae633914385 public

parent | child
Description:
Merge pull request #1 from jittat/master update from jittat/master
Commit status:
[Not Reviewed]
References:
merge default
Diff options:
Raw Diff | Patch Diff | Download Diff | Show whitespace | Increase context
Comments:
0 Commit comments 0 Inline Comments
Unresolved TODOs:
There are no unresolved TODOs
Side by Side Unified
Expand All Files Collapse All Files Wide Mode Diff
Add another comment
Browse Files

r210:3ae633914385 - - 7 files changed: 14 inserted, 15 deleted

Show file before | Show file after | Raw diff | Download diff | Show whitespace Increase context Show commentsHide comments
@@ -1,181 +1,180
1 1 #!/bin/sh
2 2
3 3 echo "This script will install and configure Cafe grader."
4 4
5 - echo "This will install Ruby 1.9.2 under rvm"
5 + RUBY_VERSION=2.1.2
6 + echo "This will install Ruby $RUBY_VERSION under RVM"
6 7
7 8 echo "Installing required apts"
8 9
9 10 sudo apt-get update
10 11 sudo apt-get install mysql-server mysql-client \
11 12 g++ gcc apache2 libmysqlclient15-dev build-essential \
12 13 git-core openssl libreadline6 libreadline6-dev \
13 14 zlib1g zlib1g-dev libssl-dev libyaml-dev libsqlite3-dev \
14 15 sqlite3 libxml2-dev libxslt-dev autoconf libc6-dev \
15 16 ncurses-dev automake libtool bison subversion \
16 17 pkg-config curl nodejs unzip pyflakes ruby default-jdk
17 18
18 19 echo "Installing RVM"
19 20 curl -k -L https://get.rvm.io | bash -s stable
20 21 source ~/.rvm/scripts/rvm
21 22
22 - echo "Installing Ruby 2.1.2 in RVM"
23 + echo "Installing Ruby $RUBY_VERSION in RVM"
23 24
24 - rvm install 2.1.2
25 - rvm use 2.1.2
25 + rvm install $RUBY_VERSION
26 + rvm use $RUBY_VERSION
26 27
27 28 echo "Fetching Cafe Grader from Git repositories"
28 29
29 30 echo "Fetching web interface"
30 31
31 32 mkdir cafe_grader
32 33 cd cafe_grader
33 34 git clone -q git://github.com/jittat/cafe-grader-web.git web
34 35
35 36 echo "Configuring rails app"
36 37
37 38 cp web/config/application.rb.SAMPLE web/config/application.rb
38 39 cp web/config/initializers/cafe_grader_config.rb.SAMPLE web/config/initializers/cafe_grader_config.rb
39 40
40 41 #replace UTC in application.rb with the system timezone
41 42 timezone='UTC'
42 43 if [ -f '/etc/timezone' ]; then
43 44 timezone=\"`cat /etc/timezone`\"
44 45 else
45 46 if [ -f '/etc/sysconfig/clock' ]; then
46 47 timezone=`grep -e '^TIMEZONE' /etc/sysconfig/clock | grep -o -e '\".*\"'`
47 48 fi
48 49 fi
49 50 replace="s!'UTC'!$timezone!g"
50 51 sed -i $replace web/config/application.rb
51 52
52 53 echo "At this point we will need MySQL user and database."
53 54 echo "Have you created MySQL user and database for Cafe grader? (Y/N) "
54 55 read ch
55 56
56 57 if [ "$ch" = "n" -o "$ch" = "N" ]
57 58 then
58 59 echo "Please open another terminal and create the user and database for Cafe grader."
59 60 echo "Don't forget to grant access to that database for the user."
60 61 echo "Please have username, password, and database name ready before continue."
61 62 echo
62 63 echo "The following are instructions:"
63 64 echo "1. Run mysql:"
64 65 echo
65 66 echo " mysql -u root -p"
66 67 echo
67 68 echo " if you have just installed mysql, the root password is the one that you have just entered"
68 69 echo "2. Create a new database, a new user, and grant access to grader database:"
69 70 echo
70 71 echo " create user 'USERNAME'@'localhost' identified by 'PASSWORD';"
71 72 echo " create database \`DATABASENEME\`;"
72 73 echo " grant all on \`DATABASENAME\`.* to 'USERNAME'@'localhost';"
73 74 echo
74 75 echo " Replace USERNAME, PASSWORD, and DATABASENAME accordingly."
75 76 echo
76 77 echo "Hit enter when ready..."
77 78 read dummy
78 79 fi
79 80
80 81 CAFE_PATH=`pwd`
81 82
82 83 cd web
83 84
84 85 echo "Please provide grader database:"
85 86 read database
86 87
87 88 echo "Please provide grader username:"
88 89 read username
89 90
90 91 echo "Please provide $username password:"
91 92 read password
92 93
93 94 echo "development:" > config/database.yml
94 95 echo " adapter: mysql2" >> config/database.yml
95 96 echo " encoding: utf8" >> config/database.yml
96 97 echo " reconnect: false" >> config/database.yml
97 98 echo " database: $database" >> config/database.yml
98 99 echo " pool: 5" >> config/database.yml
99 100 echo " username: $username" >> config/database.yml
100 101 echo " password: $password" >> config/database.yml
101 102 echo " host: localhost" >> config/database.yml
102 103 echo " socket: /var/run/mysqld/mysqld.sock" >> config/database.yml
103 104 echo "" >> config/database.yml
104 105 echo "production:" >> config/database.yml
105 106 echo " adapter: mysql2" >> config/database.yml
106 107 echo " encoding: utf8" >> config/database.yml
107 108 echo " reconnect: false" >> config/database.yml
108 109 echo " database: $database" >> config/database.yml
109 110 echo " pool: 5" >> config/database.yml
110 111 echo " username: $username" >> config/database.yml
111 112 echo " password: $password" >> config/database.yml
112 113 echo " host: localhost" >> config/database.yml
113 114 echo " socket: /var/run/mysqld/mysqld.sock" >> config/database.yml
114 115
115 116 echo "Object.instance_eval{remove_const :GRADER_ROOT_DIR}" >> config/initializers/cafe_grader_config.rb
116 117 echo "Object.instance_eval{remove_const :GRADING_RESULT_DIR}" >> config/initializers/cafe_grader_config.rb
117 118 echo "GRADER_ROOT_DIR = '$CAFE_PATH/judge'" >> config/initializers/cafe_grader_config.rb
118 119 echo "GRADING_RESULT_DIR = '$CAFE_PATH/judge/result'" >> config/initializers/cafe_grader_config.rb
119 120
120 121 echo "Installing required gems"
121 122 gem install bundler
122 123 bundle install
123 124
124 125 echo "Running rake tasks to initialize database"
125 126
126 127 rake db:migrate
127 128 rake db:seed
128 129
129 130 echo "Running rake tasks to precompile the assets"
130 131
131 132 rake assets:precompile
132 133
133 134 echo "Intalling web interface complete..."
134 135 echo
135 136 echo "Fetching grader"
136 137
137 138 cd ..
138 139
139 140 mkdir judge
140 141 cd judge
141 142 git clone -q git://github.com/jittat/cafe-grader-judge-scripts.git scripts
142 143 mkdir raw
143 144 mkdir ev-exam
144 145 mkdir ev
145 146 mkdir result
146 147 mkdir log
147 148
148 149 echo "Configuring grader"
149 150
150 151 cp scripts/config/env_exam.rb.SAMPLE scripts/config/env_exam.rb
151 152 cp scripts/config/env_grading.rb.SAMPLE scripts/config/env_grading.rb
152 153
153 154 # create new environment.rb file
154 155 echo "RAILS_ROOT = '$CAFE_PATH/web'" > scripts/config/environment.rb
155 156 echo "GRADER_ROOT = '$CAFE_PATH/judge/scripts'" >> scripts/config/environment.rb
156 157 echo "require File.join(File.dirname(__FILE__),'../lib/boot')" >> scripts/config/environment.rb
157 158 echo "require File.dirname(__FILE__) + \"/env_#{GRADER_ENV}.rb\"" >> scripts/config/environment.rb
158 159
159 160 # compiling box
160 161 MACHINE_TYPE=`uname -m`
161 162 if [ ${MACHINE_TYPE} == 'x86_64' ]; then
162 163 gcc -std=c99 -o scripts/std-script/box scripts/std-script/box64-new.c
163 - # 64-bit stuff here
164 164 else
165 165 g++ -o scripts/std-script/box scripts/std-script/box.cc
166 - # 32-bit stuff here
167 166 fi
168 167
169 168
170 169 cd ..
171 170
172 171 echo "Now you are ready to run cafe grader...."
173 172 echo
174 173 echo "Try:"
175 174 echo
176 175 echo " cd web"
177 176 echo " rails s"
178 177 echo
179 178 echo "and access web at http://localhost:3000/"
180 179 echo "The root username is 'root', its password is 'ioionrails'."
181 180
Show file before | Show file after | Raw diff | Download diff | Show whitespace Increase context Show commentsHide comments
@@ -1,73 +1,73
1 - #!/usr/bin/ruby
1 + #!/usr/bin/env ruby
2 2
3 3 # new_problem:
4 4 # * creates a directory for a problem in the current directory,
5 5 # * create standard testcase config file
6 6
7 7 require 'erb'
8 8
9 9 def process_options(options)
10 10 i = 2
11 11 while i<ARGV.length
12 12 if ARGV[i]=='-t'
13 13 options[:time_limit] = ARGV[i+1].to_i if ARGV.length>i+1
14 14 i += 1
15 15 end
16 16 if ARGV[i]=='-m'
17 17 options[:mem_limit] = ARGV[i+1].to_i if ARGV.length>i+1
18 18 i += 1
19 19 end
20 20 i += 1
21 21 end
22 22 end
23 23
24 24
25 25 puts "This script is out of dated, shall be fixed soon"
26 26 puts "Right now, you can create raw_ev and import"
27 27 exit(0)
28 28
29 29 GRADER_DIR = File.dirname(__FILE__)
30 30
31 31 # print usage
32 32 if ARGV.length < 2
33 33 puts <<USAGE
34 34 using: new_problem problem number_of_testcase [options]
35 35 * creates a directory for a problem in the current directory,
36 36 * create standard testcase config file
37 37 * options: -t time-limit (in seconds)
38 38 -m mem-limit (in MB)
39 39 USAGE
40 40 exit(127)
41 41 end
42 42
43 43 # processing arguments
44 44 problem = ARGV[0]
45 45 num_testcases = ARGV[1].to_i
46 46
47 47 options = {:time_limit => 1, :mem_limit => 16}
48 48 process_options(options)
49 49
50 50 # start working
51 51 puts "creating directories"
52 52
53 53 system("mkdir #{problem}")
54 54 system("mkdir #{problem}/script")
55 55 system("mkdir #{problem}/test_cases")
56 56
57 57 puts "creating testcases directories"
58 58
59 59 1.upto(num_testcases) do |i|
60 60 system("mkdir #{problem}/test_cases/#{i}")
61 61 end
62 62
63 63 # generating all_tests.cfg
64 64 puts "generating testcase config file"
65 65
66 66 template = File.open(File.dirname(__FILE__) + "/templates/all_tests.cfg.erb").read
67 67 all_test_cfg = ERB.new(template)
68 68
69 69 cfg_file = File.open("#{problem}/test_cases/all_tests.cfg","w")
70 70 cfg_file.puts all_test_cfg.result
71 71 cfg_file.close
72 72
73 73 puts "done"
Show file before | Show file after | Raw diff | Download diff | Show whitespace Increase context Show commentsHide comments
@@ -1,48 +1,48
1 - #!/usr/bin/ruby
1 + #!/usr/bin/env ruby
2 2
3 3 #
4 4 # This is a check script wrapper. It read all required information
5 5 # and call a real check script call REAL_CHECK_SCRIPT in directory
6 6 # [problem_home]/script
7 7 #
8 8
9 9 REAL_CHECK_SCRIPT = "<%= script_name %>"
10 10
11 11 # The REAL_CHECK_SCRIPT is called with:
12 12 #
13 13 # (script) <lang> <test-num> <in-file> <out-file> <ans-file> <full-score>
14 14 #
15 15 # and REAL_CHECK_SCRIPT's output to standard out is redirected to
16 16 # 'check_result' as required by normal check script.
17 17
18 18 problem_home = ENV['PROBLEM_HOME']
19 19 require "#{problem_home}/script/test_dsl.rb"
20 20
21 21 if ARGV.length < 2
22 22 puts "Usage: check <language> <test-number> [<output-file>]"
23 23 exit(0)
24 24 end
25 25
26 26 language = ARGV[0]
27 27 test_num = ARGV[1].to_i
28 28 if ARGV.length >= 3
29 29 output_file_name = ARGV[2]
30 30 else
31 31 output_file_name = "output.txt"
32 32 end
33 33
34 34 load "#{problem_home}/test_cases/all_tests.cfg"
35 35 problem = Problem.get_instance
36 36
37 37 answer_file_name = "#{problem_home}/test_cases/#{test_num}/answer-#{test_num}.txt"
38 38 input_file_name = "#{problem_home}/test_cases/#{test_num}/input-#{test_num}.txt"
39 39
40 40 score = problem.get_score(test_num)
41 41
42 42 cmd = "#{problem_home}/script/#{REAL_CHECK_SCRIPT} " +
43 43 "#{language} #{test_num} #{input_file_name} #{output_file_name} " +
44 44 "#{answer_file_name} #{score} > check_result"
45 45
46 46 #puts "wrapper-CMD: #{cmd}"
47 47
48 48 system(cmd)
Show file before | Show file after | Raw diff | Download diff | Show whitespace Increase context Show commentsHide comments
@@ -1,59 +1,59
1 - #!/usr/bin/ruby
1 + #!/usr/bin/env ruby
2 2
3 3 problem_home = ENV['PROBLEM_HOME']
4 4 require "#{problem_home}/script/test_dsl.rb"
5 5
6 6 if ARGV.length < 2
7 7 puts "Usage: check <language> <test-number> [<output-file>]"
8 8 exit(0)
9 9 end
10 10
11 11 language = ARGV[0]
12 12 test_num = ARGV[1].to_i
13 13 if ARGV.length >= 3
14 14 output_file_name = ARGV[2]
15 15 else
16 16 output_file_name = "output.txt"
17 17 end
18 18
19 19 load "#{problem_home}/test_cases/all_tests.cfg"
20 20 problem = Problem.get_instance
21 21
22 22 output_file = File.new(output_file_name, "r")
23 23 answer_file = File.new("#{problem_home}/test_cases/#{test_num}/answer-#{test_num}.txt")
24 24 result_file = File.new("check_result", "w")
25 25
26 26 output_file_content = output_file.read
27 27 answer_file_content = answer_file.read
28 28
29 29 report_correct = lambda {
30 30 result_file.write "Correct\n"
31 31 result_file.write problem.get_score(test_num)
32 32 result_file.write "\n"
33 33 result_file.close
34 34 exit(0)
35 35 }
36 36
37 37 report_wrong = lambda {
38 38 result_file.write "Incorrect\n"
39 39 result_file.write "0\n"
40 40 result_file.close
41 41 exit(0)
42 42 }
43 43
44 44 ##################
45 45 # Your code here #
46 46 ##################
47 47 num_pattern = /^[0-9]*/
48 48 if (output_file_content =~ num_pattern) == nil
49 49 report_wrong.call
50 50 end
51 51
52 52 output_i = output_file_content.to_i
53 53 answer_i = answer_file_content.to_i
54 54
55 55 if output_i == answer_i
56 56 report_correct.call
57 57 else
58 58 report_wrong.call
59 59 end
Show file before | Show file after | Raw diff | Download diff | Show whitespace Increase context Show commentsHide comments
@@ -1,59 +1,59
1 - #!/usr/bin/ruby
1 + #!/usr/bin/env ruby
2 2
3 3 problem_home = ENV['PROBLEM_HOME']
4 4 require "#{problem_home}/script/test_dsl.rb"
5 5
6 6 if ARGV.length < 2
7 7 puts "Usage: check <language> <test-number> [<output-file>]"
8 8 exit(0)
9 9 end
10 10
11 11 language = ARGV[0]
12 12 test_num = ARGV[1].to_i
13 13 if ARGV.length >= 3
14 14 output_file_name = ARGV[2]
15 15 else
16 16 output_file_name = "output.txt"
17 17 end
18 18
19 19 load "#{problem_home}/test_cases/all_tests.cfg"
20 20 problem = Problem.get_instance
21 21
22 22 output_file = File.new(output_file_name, "r")
23 23 answer_file = File.new("#{problem_home}/test_cases/#{test_num}/answer-#{test_num}.txt")
24 24 result_file = File.new("check_result", "w")
25 25
26 26 output_file_content = output_file.read
27 27 answer_file_content = answer_file.read
28 28
29 29 report_correct = lambda {
30 30 result_file.write "Correct\n"
31 31 result_file.write problem.get_score(test_num)
32 32 result_file.write "\n"
33 33 result_file.close
34 34 exit(0)
35 35 }
36 36
37 37 report_wrong = lambda {
38 38 result_file.write "Incorrect\n"
39 39 result_file.write "0\n"
40 40 result_file.close
41 41 exit(0)
42 42 }
43 43
44 44 ##################
45 45 # Your code here #
46 46 ##################
47 47 num_pattern = /^[0-9]*/
48 48 if (output_file_content =~ num_pattern) == nil
49 49 report_wrong.call
50 50 end
51 51
52 52 output_i = output_file_content.to_i
53 53 answer_i = answer_file_content.to_i
54 54
55 55 if output_i == answer_i
56 56 report_correct.call
57 57 else
58 58 report_wrong.call
59 59 end
Show file before | Show file after | Raw diff | Download diff | Show whitespace Increase context Show commentsHide comments
@@ -1,59 +1,59
1 - #!/usr/bin/ruby
1 + #!/usr/bin/env ruby
2 2
3 3 problem_home = ENV['PROBLEM_HOME']
4 4 require "#{problem_home}/script/test_dsl.rb"
5 5
6 6 if ARGV.length < 2
7 7 puts "Usage: check <language> <test-number> [<output-file>]"
8 8 exit(0)
9 9 end
10 10
11 11 language = ARGV[0]
12 12 test_num = ARGV[1].to_i
13 13 if ARGV.length >= 3
14 14 output_file_name = ARGV[2]
15 15 else
16 16 output_file_name = "output.txt"
17 17 end
18 18
19 19 load "#{problem_home}/test_cases/all_tests.cfg"
20 20 problem = Problem.get_instance
21 21
22 22 output_file = File.new(output_file_name, "r")
23 23 answer_file = File.new("#{problem_home}/test_cases/#{test_num}/answer-#{test_num}.txt")
24 24 result_file = File.new("check_result", "w")
25 25
26 26 output_file_content = output_file.read
27 27 answer_file_content = answer_file.read
28 28
29 29 report_correct = lambda {
30 30 result_file.write "Correct\n"
31 31 result_file.write problem.get_score(test_num)
32 32 result_file.write "\n"
33 33 result_file.close
34 34 exit(0)
35 35 }
36 36
37 37 report_wrong = lambda {
38 38 result_file.write "Incorrect\n"
39 39 result_file.write "0\n"
40 40 result_file.close
41 41 exit(0)
42 42 }
43 43
44 44 ##################
45 45 # Your code here #
46 46 ##################
47 47 num_pattern = /^[0-9]*/
48 48 if (output_file_content =~ num_pattern) == nil
49 49 report_wrong.call
50 50 end
51 51
52 52 output_i = output_file_content.to_i
53 53 answer_i = answer_file_content.to_i
54 54
55 55 if output_i == answer_i
56 56 report_correct.call
57 57 else
58 58 report_wrong.call
59 59 end
Show file before | Show file after | Raw diff | Download diff | Show whitespace Increase context Show commentsHide comments
@@ -13,316 +13,316
13 13 :id => 1, :name => 'test_normal',
14 14 :full_score => 135)
15 15 @user_user1 = stub(User,
16 16 :id => 1, :login => 'user1')
17 17
18 18 @engine = Grader::Engine.new
19 19 init_sandbox
20 20 end
21 21
22 22 it "should grade normal submission" do
23 23 grader_should(:grade => "test1_correct.c",
24 24 :on => @problem_test_normal,
25 25 :and_report => {
26 26 :score => 135,
27 27 :comment => /^(\[|P|\])+/})
28 28 end
29 29
30 30
31 31 it "should grade normal submission in C++" do
32 32 cpp_lang = stub(Language, :name => 'cpp', :ext => 'cpp')
33 33
34 34 grader_should(:grade => "test1_correct_cc.cc",
35 35 :language => cpp_lang,
36 36 :on => @problem_test_normal,
37 37 :and_report => {
38 38 :score => 135,
39 39 :comment => /^(\[|P|\])+/})
40 40 end
41 41
42 42 it "should produce error message when submission cannot compile" do
43 43 grader_should(:grade => "test1_compile_error.c",
44 44 :on => @problem_test_normal,
45 45 :and_report => {
46 46 :score => 0,
47 47 :comment => 'compilation error',
48 48 :compiler_message => /[Ee]rror/})
49 49 end
50 50
51 51 it "should produce timeout error when submission runs forever" do
52 52 @problem_test_timeout = stub(Problem,
53 53 :id => 1, :name => 'test_timeout',
54 54 :full_score => 10)
55 55 grader_should(:grade => "test2_timeout.c",
56 56 :on => @problem_test_timeout,
57 57 :and_report => {
58 58 :score => 0,
59 59 :comment => 'TT'})
60 60 end
61 61
62 62 it "should produce timeout error correctly with fractional running time and fractional time limits" do
63 63 @problem_test_timeout = stub(Problem,
64 64 :id => 1, :name => 'test_timeout',
65 65 :full_score => 20)
66 66 grader_should(:grade => "test2_05sec.c",
67 67 :on => @problem_test_timeout,
68 68 :and_report => {
69 69 :score => 10,
70 70 :comment => 'TP'})
71 71 end
72 72
73 73 it "should produce runtime error when submission uses too much static memory" do
74 74 @problem_test_memory = stub(Problem,
75 75 :id => 1, :name => 'test_memory',
76 76 :full_score => 20)
77 77 grader_should(:grade => "add_too_much_memory_static.c",
78 78 :on => @problem_test_memory,
79 79 :and_report => {
80 80 :score => 10,
81 81 :comment => /[^P]P/})
82 82 end
83 83
84 84 it "should not allow submission to allocate too much dynamic memory" do
85 85 @problem_test_memory = stub(Problem,
86 86 :id => 1, :name => 'test_memory',
87 87 :full_score => 20)
88 88 grader_should(:grade => "add_too_much_memory_dynamic.c",
89 89 :on => @problem_test_memory,
90 90 :and_report => {
91 91 :score => 10,
92 92 :comment => /[^P]P/})
93 93 end
94 94
95 95 it "should score test runs correctly when submission fails in some test case" do
96 96 grader_should(:grade => "add_fail_test_case_1.c",
97 97 :on => @problem_test_normal,
98 98 :and_report => {
99 99 :score => 105,
100 100 :comment => /^.*(-|x).*$/})
101 101 end
102 102
103 103 it "should fail submission with non-zero exit status" do
104 104 grader_should(:grade => "add_nonzero_exit_status.c",
105 105 :on => @problem_test_normal,
106 106 :and_report => {
107 107 :score => 0,
108 108 :comment => /^(\[|x|\])+$/})
109 109 end
110 110
111 111 it "should not allow malicious submission to see PROBLEM_HOME" do
112 112 problem_test_yesno = stub(Problem,
113 113 :id => 1, :name => 'test_yesno',
114 114 :full_score => 10)
115 115 grader_should(:grade => "yesno_access_problem_home.c",
116 116 :on => problem_test_yesno,
117 117 :and_report => {
118 118 :score => 0,
119 119 :comment => /(-|x)/})
120 120 end
121 121
122 122 it "should not allow malicious submission to open files" do
123 123 problem_test_yesno = stub(Problem,
124 124 :id => 1, :name => 'test_yesno',
125 125 :full_score => 10)
126 126 grader_should(:grade => "yesno_open_file.c",
127 127 :on => problem_test_yesno,
128 128 :and_report => {
129 129 :score => 0,
130 130 :comment => /(-|x)/})
131 131 end
132 132
133 133 def grader_should(args)
134 134 @user1 = stub(User,
135 135 :id => 1, :login => 'user1')
136 136
137 137 submission =
138 138 create_submission_from_file(1, @user1, args[:on], args[:grade], args[:language])
139 139 submission.should_receive(:graded_at=)
140 140
141 141 expected_score = args[:and_report][:score]
142 142 expected_comment = args[:and_report][:comment]
143 143 if args[:and_report][:compiler_message]!=nil
144 144 expected_compiler_message = args[:and_report][:compiler_message]
145 145 else
146 146 expected_compiler_message = ''
147 147 end
148 148
149 149 submission.should_receive(:points=).with(expected_score)
150 150 submission.should_receive(:grader_comment=).with(expected_comment)
151 151 submission.should_receive(:compiler_message=).with(expected_compiler_message)
152 152 submission.should_receive(:save)
153 153
154 154 @engine.grade(submission)
155 155 end
156 156
157 157 protected
158 158
159 159 def create_normal_submission_mock_from_file(source_fname)
160 160 create_submission_from_file(1, @user_user1, @problem_test_normal, source_fname)
161 161 end
162 162
163 163 end
164 164
165 165 describe "A grader engine, when grading test requests" do
166 166
167 167 include GraderEngineHelperMethods
168 168
169 169 before(:each) do
170 170 @config = Grader::Configuration.get_instance
171 171 @engine = Grader::Engine.new(:room_maker => Grader::TestRequestRoomMaker.new,
172 172 :reporter => Grader::TestRequestReporter.new)
173 173 init_sandbox
174 174 end
175 175
176 176 it "should report error if there is no problem template" do
177 177 problem = stub(Problem,
178 178 :id => 1, :name => 'nothing')
179 179 grader_should(:grade => 'test1_correct.c',
180 180 :on => problem,
181 181 :with => 'in1.txt',
182 182 :and_report => {
183 183 :graded_at= => nil,
184 184 :compiler_message= => '',
185 185 :grader_comment= => '',
186 186 :running_stat= => /template not found/,
187 187 :running_time= => nil,
188 188 :exit_status= => nil,
189 189 :memory_usage= => nil,
190 190 :save => nil})
191 191 end
192 192
193 193 it "should run test request and produce output file" do
194 194 problem = stub(Problem,
195 195 :id => 1, :name => 'test_normal')
196 196 grader_should(:grade => 'test1_correct.c',
197 197 :on => problem,
198 198 :with => 'in1.txt',
199 199 :and_report => {
200 200 :graded_at= => nil,
201 201 :compiler_message= => '',
202 202 :grader_comment= => '',
203 203 :running_stat= => /0.0\d* sec./,
204 204 :output_file_name= => lambda { |fname|
205 - File.exists?(fname).should be_true
205 + File.exists?(fname).should be true
206 206 },
207 207 :running_time= => nil,
208 208 :exit_status= => nil,
209 209 :memory_usage= => nil,
210 210 :save => nil})
211 211 end
212 212
213 213 it "should clean up problem directory after running test request" do
214 214 problem = stub(Problem,
215 215 :id => 1, :name => 'test_normal')
216 216 grader_should(:grade => 'test1_correct.c',
217 217 :on => problem,
218 218 :with => 'in1.txt',
219 219 :and_report => {
220 220 :graded_at= => nil,
221 221 :compiler_message= => '',
222 222 :grader_comment= => '',
223 223 :running_stat= => nil,
224 224 :output_file_name= => nil,
225 225 :running_time= => nil,
226 226 :exit_status= => nil,
227 227 :memory_usage= => nil,
228 228 :save => nil})
229 - File.exists?(@config.user_result_dir + "/test_request/test_normal/test_cases/1/input-1.txt").should be_false
229 + File.exists?(@config.user_result_dir + "/test_request/test_normal/test_cases/1/input-1.txt").should be false
230 230 end
231 231
232 232 it "should compile test request with error and report compilation error" do
233 233 problem = stub(Problem,
234 234 :id => 1, :name => 'test_normal')
235 235 grader_should(:grade => 'test1_compile_error.c',
236 236 :on => problem,
237 237 :with => 'in1.txt',
238 238 :and_report => {
239 239 :graded_at= => nil,
240 240 :compiler_message= => /.+/,
241 241 :grader_comment= => /[Cc]ompil.*error/,
242 242 :running_stat= => '',
243 243 :save => nil})
244 244 end
245 245
246 246 it "should report exit status" do
247 247 problem = stub(Problem,
248 248 :id => 1, :name => 'test_normal')
249 249 grader_should(:grade => 'add_nonzero_exit_status.c',
250 250 :on => problem,
251 251 :with => 'in1.txt',
252 252 :and_report => {
253 253 :graded_at= => nil,
254 254 :compiler_message= => '',
255 255 :grader_comment= => '',
256 256 :running_stat= => /[Ee]xit.*status.*10.*0\.0\d* sec/m,
257 257 :output_file_name= => lambda { |fname|
258 - File.exists?(fname).should be_true
258 + File.exists?(fname).should be true
259 259 },
260 260 :running_time= => nil,
261 261 :exit_status= => /10/,
262 262 :memory_usage= => nil,
263 263 :save => nil})
264 264 end
265 265
266 266 it "should produce running statistics for normal submission" do
267 267 problem = stub(Problem,
268 268 :id => 1, :name => 'test_normal')
269 269 grader_should(:grade => 'test_run_stat.c',
270 270 :on => problem,
271 271 :with => 'in1.txt',
272 272 :and_report => {
273 273 :graded_at= => nil,
274 274 :compiler_message= => '',
275 275 :grader_comment= => '',
276 276 :running_stat= => nil,
277 277 :output_file_name= => lambda { |fname|
278 - File.exists?(fname).should be_true
278 + File.exists?(fname).should be true
279 279 },
280 280 :running_time= => lambda { |t|
281 281 (t>=0.14) and (t<=0.16)
282 282 },
283 283 :exit_status= => nil,
284 284 :memory_usage= => lambda { |s|
285 285 (s>=500) and (s<=1000)
286 286 },
287 287 :save => nil})
288 288 end
289 289
290 290 protected
291 291 def grader_should(args)
292 292 @user1 = stub(User,
293 293 :id => 1, :login => 'user1')
294 294
295 295 problem = args[:on]
296 296 input_file = @config.test_request_input_base_dir + "/" + args[:with]
297 297
298 298 submission =
299 299 create_submission_from_file(1, @user1, args[:on], args[:grade])
300 300
301 301 test_request = stub(TestRequest,
302 302 :id => 1,
303 303 :user => @user1,
304 304 :problem => problem,
305 305 :submission => submission,
306 306 :input_file_name => input_file,
307 307 :language => submission.language,
308 308 :problem_name => problem.name)
309 309
310 310 expectations = args[:and_report]
311 311
312 312 expectations.each do |key,val|
313 313 if val==nil
314 314 test_request.should_receive(key)
315 315 elsif val.class == Proc
316 316 test_request.should_receive(key) { |fname|
317 317 val.call(fname)
318 318 }
319 319 else
320 320 test_request.should_receive(key).with(val)
321 321 end
322 322 end
323 323
324 324 @engine.grade(test_request)
325 325 end
326 326
327 327 end
328 328
You need to be logged in to leave comments. Login now