cafe-grader-judge-scripts

Commit r254:8d92eda43960 public

parent | child
Description:
- new install script - add comment in grader chain
Commit status:
[Not Reviewed]
References:
default
Diff options:
Raw Diff | Patch Diff | Download Diff | Ignore whitespace | Increase context
Comments:
0 Commit comments 0 Inline Comments
Unresolved TODOs:
There are no unresolved TODOs
Side by Side Unified
Expand All Files Collapse All Files Wide Mode Diff
Add another comment
Browse Files

r254:8d92eda43960 - - 4 files changed: 48 inserted, 41 deleted

Show file before | Show file after | Raw diff | Download diff | Ignore whitespace Increase context Show commentsHide comments
@@ -1,59 +1,44
1 1 #!/bin/sh
2 2
3 + #installation script for cafe-grader, for ubuntu 16.04
4 +
3 5 echo "This script will install and configure Cafe grader."
4 6
5 - RUBY_VERSION=2.1.2
6 - echo "This will install Ruby $RUBY_VERSION under RVM"
7 -
8 - echo "Installing required apts"
9 -
10 - sudo apt-get update
11 - sudo apt-get install mysql-server mysql-client \
12 - g++ gcc apache2 libmysqlclient20 build-essential \
13 - git-core openssl libreadline6 libreadline6-dev \
14 - zlib1g zlib1g-dev libssl-dev libyaml-dev libsqlite3-dev \
15 - sqlite3 libxml2-dev libxslt-dev autoconf libc6-dev \
16 - ncurses-dev automake libtool bison subversion \
17 - pkg-config curl nodejs unzip pyflakes ruby default-jdk \
18 - libmysqld-dev mercurial python-setuptools python-dev python3-numpy
19 -
20 - echo "Installing RVM"
21 - curl -k -L https://get.rvm.io | bash -s stable
22 - source ~/.rvm/scripts/rvm
7 + RUBY_VERSION=2.3.7
23 8
24 9 echo "Installing Ruby $RUBY_VERSION in RVM"
25 10
26 11 rvm install $RUBY_VERSION
27 12 rvm use $RUBY_VERSION
28 13
29 14 echo "Fetching Cafe Grader from Git repositories"
30 15
31 16 echo "Fetching web interface"
32 17
33 18 mkdir cafe_grader
34 19 cd cafe_grader
35 - git clone -q git://github.com/jittat/cafe-grader-web.git web
20 + git clone -q git://github.com/cafe-grader-team/cafe-grader-web.git web
36 21
37 22 echo "Configuring rails app"
38 23
39 24 cp web/config/application.rb.SAMPLE web/config/application.rb
40 25 cp web/config/initializers/cafe_grader_config.rb.SAMPLE web/config/initializers/cafe_grader_config.rb
41 26
42 27 #replace UTC in application.rb with the system timezone
43 28 timezone='UTC'
44 29 if [ -f '/etc/timezone' ]; then
45 30 timezone=\"`cat /etc/timezone`\"
46 31 else
47 32 if [ -f '/etc/sysconfig/clock' ]; then
48 33 timezone=`grep -e '^TIMEZONE' /etc/sysconfig/clock | grep -o -e '\".*\"'`
49 34 fi
50 35 fi
51 36 replace="s!'UTC'!$timezone!g"
52 37 sed -i $replace web/config/application.rb
53 38
54 39 echo "At this point we will need MySQL user and database."
55 40 echo "Have you created MySQL user and database for Cafe grader? (Y/N) "
56 41 read ch
57 42
58 43 if [ "$ch" = "n" -o "$ch" = "N" ]
59 44 then
@@ -111,57 +96,68
111 96 echo " pool: 5" >> config/database.yml
112 97 echo " username: $username" >> config/database.yml
113 98 echo " password: $password" >> config/database.yml
114 99 echo " host: localhost" >> config/database.yml
115 100 echo " socket: /var/run/mysqld/mysqld.sock" >> config/database.yml
116 101
117 102 echo "Object.instance_eval{remove_const :GRADER_ROOT_DIR}" >> config/initializers/cafe_grader_config.rb
118 103 echo "Object.instance_eval{remove_const :GRADING_RESULT_DIR}" >> config/initializers/cafe_grader_config.rb
119 104 echo "GRADER_ROOT_DIR = '$CAFE_PATH/judge'" >> config/initializers/cafe_grader_config.rb
120 105 echo "GRADING_RESULT_DIR = '$CAFE_PATH/judge/result'" >> config/initializers/cafe_grader_config.rb
121 106
122 107 echo "Installing required gems"
123 108 gem install bundler
124 109 bundle install
125 110
126 111 echo "Running rake tasks to initialize database"
127 112
128 113 rake db:migrate
129 114 rake db:seed
130 115
131 116 echo "Running rake tasks to precompile the assets"
132 117
133 118 rake assets:precompile
134 119
120 + echo "setup the secret file"
121 + SECRET_A=`rake secret`
122 + SECRET_B=`rake secret`
123 + SECRET_C=`rake secret`
124 + echo "development:" > config/secrets.yml
125 + echo " secret_key_base: '$SECRET_A'" >> config/secrets.yml
126 + echo "test:" >> config/secrets.yml
127 + echo " secret_key_base: '$SECRET_B'" >> config/secrets.yml
128 + echo "production:" >> config/secrets.yml
129 + echo " secret_key_base: '$SECRET_C'" >> config/secrets.yml
130 +
135 131 echo "Intalling web interface complete..."
136 132 echo
137 133 echo "Fetching grader"
138 134
139 135 cd ..
140 136
141 137 mkdir judge
142 138 cd judge
143 - git clone -q git://github.com/jittat/cafe-grader-judge-scripts.git scripts
139 + git clone -q git://github.com/cafe-grader-team/cafe-grader-judge-scripts.git scripts
144 140 mkdir raw
145 141 mkdir ev-exam
146 142 mkdir ev
147 143 mkdir result
148 144 mkdir log
149 145
150 146 echo "Configuring grader"
151 147
152 148 cp scripts/config/env_exam.rb.SAMPLE scripts/config/env_exam.rb
153 149 cp scripts/config/env_grading.rb.SAMPLE scripts/config/env_grading.rb
154 150
155 151 # create new environment.rb file
156 152 echo "RAILS_ROOT = '$CAFE_PATH/web'" > scripts/config/environment.rb
157 153 echo "GRADER_ROOT = '$CAFE_PATH/judge/scripts'" >> scripts/config/environment.rb
158 154 echo "require File.join(File.dirname(__FILE__),'../lib/boot')" >> scripts/config/environment.rb
159 155 echo "require File.dirname(__FILE__) + \"/env_#{GRADER_ENV}.rb\"" >> scripts/config/environment.rb
160 156
161 157 # compiling box
162 158 MACHINE_TYPE=`uname -m`
163 159 if [ ${MACHINE_TYPE} == 'x86_64' ]; then
164 160 gcc -std=c99 -o scripts/std-script/box scripts/std-script/box64-new.c
165 161 else
166 162 g++ -o scripts/std-script/box scripts/std-script/box.cc
167 163 fi
Show file before | Show file after | Raw diff | Download diff | Ignore whitespace Increase context Show commentsHide comments
@@ -1,54 +1,54
1 1 require 'fileutils'
2 2 require File.join(File.dirname(__FILE__),'dir_init')
3 3
4 4 module Grader
5 5
6 6 #
7 7 # A grader engine grades a submission, against anything: a test
8 8 # data, or a user submitted test data. It uses two helpers objects:
9 9 # room_maker and reporter.
10 10 #
11 11 class Engine
12 -
12 +
13 13 attr_writer :room_maker
14 14 attr_writer :reporter
15 15
16 16 def initialize(options={})
17 17 # default options
18 18 if not options.include? :room_maker
19 19 options[:room_maker] = Grader::SubmissionRoomMaker.new
20 20 end
21 21 if not options.include? :reporter
22 22 options[:reporter] = Grader::SubmissionReporter.new
23 23 end
24 24
25 25 @config = Grader::Configuration.get_instance
26 26
27 27 @room_maker = options[:room_maker]
28 28 @reporter = options[:reporter]
29 29 end
30 -
30 +
31 31 # takes a submission, asks room_maker to produce grading directories,
32 32 # calls grader scripts, and asks reporter to save the result
33 33 def grade(submission)
34 34 current_dir = FileUtils.pwd
35 35
36 36 user = submission.user
37 37 problem = submission.problem
38 38
39 39 begin
40 40 # TODO: will have to create real exception for this
41 41 if user==nil or problem == nil
42 42 @reporter.report_error(submission,"Grading error: problem with submission")
43 43 raise "engine: user or problem is nil"
44 44 end
45 45
46 46 # TODO: this is another hack so that output only task can be judged
47 47 if submission.language!=nil
48 48 language = submission.language.name
49 49 lang_ext = submission.language.ext
50 50 else
51 51 language = 'c'
52 52 lang_ext = 'c'
53 53 end
54 54
@@ -56,134 +56,142
56 56 # only look for c++.
57 57 if language == 'cpp'
58 58 language = 'c++'
59 59 end
60 60
61 61 # COMMENT: should it be only source.ext?
62 62 if problem!=nil
63 63 source_name = "#{problem.name}.#{lang_ext}"
64 64 else
65 65 source_name = "source.#{lang_ext}"
66 66 end
67 67
68 68 grading_dir = @room_maker.produce_grading_room(submission)
69 69 @room_maker.save_source(submission,source_name)
70 70 problem_home = @room_maker.find_problem_home(submission)
71 71
72 72 # puts "GRADING DIR: #{grading_dir}"
73 73 # puts "PROBLEM DIR: #{problem_home}"
74 74
75 75 if !FileTest.exist?(problem_home)
76 76 puts "PROBLEM DIR: #{problem_home}"
77 77 raise "engine: No test data."
78 78 end
79 79
80 + # copy the source script, using lock
80 81 dinit = DirInit::Manager.new(problem_home)
81 82
83 + # lock the directory and copy the scripts
82 84 dinit.setup do
83 85 copy_log = copy_script(problem_home)
84 86 save_copy_log(problem_home,copy_log)
85 87 end
86 -
88 +
87 89 call_judge(problem_home,language,grading_dir,source_name)
88 90
89 91 @reporter.report(submission,"#{grading_dir}/test-result")
90 92
93 + # unlock the directory
91 94 dinit.teardown do
92 95 copy_log = load_copy_log(problem_home)
93 96 clear_copy_log(problem_home)
94 97 clear_script(copy_log,problem_home)
95 98 end
96 99
97 100 rescue RuntimeError => msg
98 101 @reporter.report_error(submission, msg)
99 102 puts "ERROR: #{msg}"
100 103
101 104 ensure
102 105 @room_maker.clean_up(submission)
103 106 Dir.chdir(current_dir) # this is really important
104 107 end
105 108 end
106 -
109 +
107 110 protected
108 -
111 +
109 112 def talk(str)
110 113 if @config.talkative
111 114 puts str
112 115 end
113 116 end
114 117
118 + #change directory to problem_home
119 + #call the "judge" script
115 120 def call_judge(problem_home,language,grading_dir,fname)
116 121 ENV['PROBLEM_HOME'] = problem_home
117 122 ENV['RUBYOPT'] = ''
118 -
123 +
119 124 talk grading_dir
120 125 Dir.chdir grading_dir
121 126 script_name = "#{problem_home}/script/judge"
122 127 cmd = "#{script_name} #{language} #{fname}"
123 128 talk "CMD: #{cmd}"
124 129 warn "ERROR: file does not exists #{script_name}" unless File.exists? script_name
125 130 system(cmd)
126 131 end
127 132
128 133 def get_std_script_dir
129 134 GRADER_ROOT + '/std-script'
130 135 end
131 136
137 + #copy any script presented in std-script directory that is not in the problem_home
138 + #this allow a problem setter to provide their own version for each script
139 + #in case that they want to hack something
132 140 def copy_script(problem_home)
133 141 script_dir = "#{problem_home}/script"
134 142 std_script_dir = get_std_script_dir
135 143
136 144 raise "engine: std-script directory not found" if !FileTest.exist?(std_script_dir)
137 145
138 146 scripts = Dir[std_script_dir + '/*']
139 -
147 +
140 148 copied = []
141 149
142 150 scripts.each do |s|
143 151 fname = File.basename(s)
144 152 next if FileTest.directory?(s)
145 153 if !FileTest.exist?("#{script_dir}/#{fname}")
146 154 copied << fname
147 155 FileUtils.cp(s, "#{script_dir}", :preserve => true)
148 156 end
149 157 end
150 -
158 +
151 159 return copied
152 160 end
153 161
154 162 def copy_log_filename(problem_home)
155 163 return File.join(problem_home, '.scripts_copied')
156 164 end
157 165
158 166 def save_copy_log(problem_home, log)
159 167 f = File.new(copy_log_filename(problem_home),"w")
160 168 log.each do |fname|
161 169 f.write("#{fname}\n")
162 170 end
163 171 f.close
164 172 end
165 -
173 +
166 174 def load_copy_log(problem_home)
167 175 f = File.new(copy_log_filename(problem_home),"r")
168 176 log = []
169 177 f.readlines.each do |line|
170 178 log << line.strip
171 179 end
172 180 f.close
173 181 log
174 182 end
175 183
176 184 def clear_copy_log(problem_home)
177 185 File.delete(copy_log_filename(problem_home))
178 186 end
179 187
180 188 def clear_script(log,problem_home)
181 189 log.each do |s|
182 190 FileUtils.rm("#{problem_home}/script/#{s}")
183 191 end
184 192 end
185 193
186 194 def mkdir_if_does_not_exist(dirname)
187 195 Dir.mkdir(dirname) if !FileTest.exist?(dirname)
188 196 end
189 197
Show file before | Show file after | Raw diff | Download diff | Ignore whitespace Increase context Show commentsHide comments
@@ -3,48 +3,51
3 3 #
4 4
5 5 module Grader
6 6
7 7 class Runner
8 8
9 9 def initialize(engine, grader_process=nil)
10 10 @engine = engine
11 11 @grader_process = grader_process
12 12 end
13 13
14 14 def grade_oldest_task
15 15 task = Task.get_inqueue_and_change_status(Task::STATUS_GRADING)
16 16 if task!=nil
17 17 @grader_process.report_active(task) if @grader_process!=nil
18 18
19 19 submission = Submission.find(task.submission_id)
20 20 @engine.grade(submission)
21 21 task.status_complete!
22 22 @grader_process.report_inactive(task) if @grader_process!=nil
23 23 end
24 24 return task
25 25 end
26 26
27 + # grade a specified problem for the latest submission of each user
28 + # optionally, on all submission when options[:all_sub] is set
29 + # optionally, only submission that has error (use when the problem itself has some problem)
27 30 def grade_problem(problem, options={})
28 31 user_index = 0
29 32 user_count = User.count
30 33 User.find_each do |u|
31 34 puts "user: #{u.login} (#{user_index}/#{user_count})"
32 35 user_index += 1
33 36 if options[:user_conditions]!=nil
34 37 con_proc = options[:user_conditions]
35 38 next if not con_proc.call(u)
36 39 end
37 40 if options[:all_sub]
38 41 Submission.where(user_id: u.id,problem_id: problem.id).find_each do |sub|
39 42 next if options[:only_err] and sub.grader_comment != 'error during grading'
40 43 @engine.grade(sub)
41 44 end
42 45 else
43 46 last_sub = Submission.find_last_by_user_and_problem(u.id,problem.id)
44 47 if last_sub!=nil
45 48 @engine.grade(last_sub) unless options[:only_err] and last_sub.grader_comment != 'error during grading'
46 49 end
47 50 end
48 51 end
49 52 end
50 53
Show file before | Show file after | Raw diff | Download diff | Ignore whitespace Increase context Show commentsHide comments
@@ -1,154 +1,154
1 1 module Grader
2 2
3 3 class SubmissionRoomMaker
4 4 def initialize
5 5 @config = Grader::Configuration.get_instance
6 6 end
7 -
7 +
8 8 def produce_grading_room(submission)
9 9 user = submission.user
10 10 problem = submission.problem
11 11 grading_room = "#{@config.user_result_dir}/" +
12 12 "#{user.login}/#{problem.name}/#{submission.id}"
13 -
13 +
14 14 FileUtils.mkdir_p(grading_room)
15 15 grading_room
16 16 end
17 -
17 +
18 18 def find_problem_home(submission)
19 19 problem = submission.problem
20 20 "#{@config.problems_dir}/#{problem.name}"
21 21 end
22 22
23 23 def save_source(submission,source_name)
24 24 dir = self.produce_grading_room(submission)
25 25 f = File.open("#{dir}/#{source_name}","w")
26 26 f.write(submission.source)
27 27 f.close
28 28 end
29 29
30 30 def clean_up(submission)
31 31 end
32 32 end
33 -
33 +
34 34 class SubmissionReporter
35 35 def initialize(options={})
36 36 options = {:dry_run => false, :result_collector => nil}.merge(options)
37 37 @config = Grader::Configuration.get_instance
38 38 @dry_run = options[:dry_run]
39 39 @result_collector = options[:result_collector]
40 40 end
41 -
41 +
42 42 def report(sub,test_result_dir)
43 43 result = read_result(test_result_dir)
44 44 if @result_collector
45 45 @result_collector.save(sub,
46 46 result)
47 47 end
48 48 save_result(sub,result)
49 49 end
50 -
50 +
51 51 def report_error(sub,msg)
52 52 save_result(sub,{:points => 0,
53 53 :comment => "Grading error: #{msg}" })
54 54 end
55 55
56 56 protected
57 57 def read_result(test_result_dir)
58 58 cmp_msg_fname = "#{test_result_dir}/compiler_message"
59 59 if FileTest.exist?(cmp_msg_fname)
60 60 cmp_file = File.open(cmp_msg_fname)
61 61 cmp_msg = cmp_file.read
62 62 cmp_file.close
63 63 else
64 64 cmp_msg = ""
65 65 end
66 -
66 +
67 67 result_fname = "#{test_result_dir}/result"
68 68 comment_fname = "#{test_result_dir}/comment"
69 69 runstat_fname = "#{test_result_dir}/run_stat"
70 70 if FileTest.exist?(result_fname)
71 71 comment = ""
72 72 begin
73 73 result_file = File.open(result_fname)
74 74 result = result_file.readline.to_i
75 75 result_file.close
76 76 rescue
77 77 result = 0
78 78 comment = "error reading result file."
79 79 end
80 -
80 +
81 81 begin
82 82 comment_file = File.open(comment_fname)
83 83 comment += comment_file.readline.chomp
84 84 comment_file.close
85 85 rescue
86 86 comment += ""
87 87 end
88 88
89 89 begin
90 90 runstat_file = File.open(runstat_fname)
91 91 max_runtime = runstat_file.readline.to_f
92 92 peak_memory = runstat_file.readline.to_i
93 93 rescue
94 94 max_runtime = -1
95 95 peak_memory = -1
96 96 end
97 97
98 98
99 99 return {points: result,
100 100 comment: comment,
101 101 cmp_msg: cmp_msg,
102 102 max_runtime: max_runtime,
103 103 peak_memory: peak_memory
104 104 }
105 105 else
106 106 if FileTest.exist?("#{test_result_dir}/a.out")
107 107 return {:points => 0,
108 108 :comment => 'error during grading',
109 109 :cmp_msg => cmp_msg}
110 110 else
111 111 return {:points => 0,
112 112 :comment => 'compilation error',
113 113 :cmp_msg => cmp_msg}
114 114 end
115 115 end
116 116 end
117 -
117 +
118 118 def save_result(submission,result)
119 119 problem = submission.problem
120 120 submission.graded_at = Time.now.gmtime
121 121 points = result[:points]
122 122 submission.points = points
123 123 comment = @config.report_comment(result[:comment])
124 124
125 125 submission.peak_memory = result[:peak_memory]
126 126 submission.max_runtime = result[:max_runtime]
127 127 submission.effective_code_length =submission.source.length
128 128
129 129 #
130 130 # TODO: FIX THIS MESSAGE
131 131 #
132 132 if problem == nil
133 133 submission.grader_comment = 'PASSED: ' + comment + '(problem is nil)'
134 134 elsif points == problem.full_score
135 135 #submission.grader_comment = 'PASSED: ' + comment
136 136 submission.grader_comment = comment
137 137 elsif result[:comment].chomp =~ /^[\[\]P]+$/
138 138 submission.grader_comment = 'PASSED: ' + comment + '(inconsistent score)'
139 139 else
140 140 #submission.grader_comment = 'FAILED: ' + comment
141 141 submission.grader_comment = comment
142 142 end
143 -
143 +
144 144 #very lazy trim the string
145 145 submission.compiler_message = result[:cmp_msg][0..60000] or ''
146 146
147 147 if not @dry_run
148 148 submission.save
149 149 end
150 150 end
151 -
151 +
152 152 end
153 -
153 +
154 154 end
You need to be logged in to leave comments. Login now