cafe-grader-judge-scripts

Commit r241:19abb7c667e3 public

parent | child
Description:
merge
Commit status:
[Not Reviewed]
References:
merge default
Diff options:
Raw Diff | Patch Diff | Download Diff | Show whitespace | Increase context
Comments:
0 Commit comments 0 Inline Comments
Unresolved TODOs:
There are no unresolved TODOs
Side by Side Unified
Expand All Files Collapse All Files Wide Mode Diff
Add another comment
Browse Files

r241:19abb7c667e3 - - 7 files changed: 124 inserted, 4 deleted

Show file before | Show file after | Raw diff | Download diff | Show whitespace Increase context Show commentsHide comments
@@ -0,0 +1,93
1 + #!/usr/bin/env ruby
2 +
3 + def config
4 + Grader::Configuration.get_instance
5 + end
6 +
7 + def display_manual
8 + puts <<USAGE
9 + Check similarity between submission
10 + using: check_similar sub1 sub2
11 + -- or --
12 + check_similar problem_name
13 + sub1 and sub2 are submission IDs to be checked
14 + problem_name will check all submissions of the problem wit problem short name is 'problem_name'
15 +
16 + The output are given as
17 + sub1.login, sub1.id, sub1.point, sub2.login, sub2.id, sub2.point, similarity
18 +
19 + USAGE
20 + end
21 +
22 + def process_options_and_stop_file
23 +
24 + # Process 'help' option
25 + if (ARGV.length == 0) or ((ARGV.length==1) and (/help/.match(ARGV[0])))
26 + display_manual
27 + exit(0)
28 + end
29 +
30 + #default options
31 + options = {
32 + :dry_run => false,
33 + }
34 +
35 +
36 + if ARGV.length == 2
37 + options[:sub1] = ARGV[0].to_i
38 + options[:sub2] = ARGV[1].to_i
39 + elsif ARGV.length == 1
40 + options[:problem] = ARGV[0]
41 + end
42 +
43 +
44 + return options
45 + end
46 +
47 + def compare(sub1,sub2,full = sub1.problem.full_score)
48 + dis = @jarow.getDistance(sub1.source, sub2.source)
49 + puts [sub1.user.login,"##{sub1.id}",(sub1.points * 100.0 / full).to_i,
50 + sub2.user.login,"##{sub2.id}",(sub2.points * 100.0 / full).to_i,
51 + "#{dis * 100}%"].join(',')
52 + end
53 +
54 + #########################################
55 + # main program
56 + #########################################
57 +
58 + options = process_options_and_stop_file
59 +
60 + # load grader environment
61 + GRADER_ENV = 'grading'
62 + require File.join(File.dirname(__FILE__),'config/environment')
63 +
64 + # boot rails, to be able to use the active record
65 + RAILS_ENV = config.rails_env
66 + require RAILS_ROOT + '/config/environment'
67 +
68 + # load comparator
69 + require 'fuzzystringmatch'
70 + @jarow = FuzzyStringMatch::JaroWinkler.create( :native )
71 +
72 + if options[:problem]
73 + p = Problem.where(name: options[:problem]).first
74 + unless p
75 + puts "cannot find problem #{options[:problem]}"
76 + exit(0)
77 + end
78 + subs = Submission.where(problem: p)
79 + full_score = p.full_score.to_i
80 + subs.each.with_index do |s1,i|
81 + puts "processing #{i+1} out of #{subs.length}"
82 + subs.each do |s2|
83 + if s1.user != s2.user
84 + compare(s1,s2,full_score)
85 + end
86 + end
87 + end
88 + else
89 + sub1 = Submission.find(options[:sub1])
90 + sub2 = Submission.find(options[:sub2])
91 + compare(sub1,sub2)
92 + end
93 +
Show file before | Show file after | Raw diff | Download diff | Show whitespace Increase context Show commentsHide comments
@@ -0,0 +1,6
1 + #!/bin/bash
2 + count=`ps aux | grep "cafe_grader" | grep "grader grading queue" | wc -l`
3 + if [ $count -lt 1 ]; then
4 + cd /home/dae/cafe_grader/judge
5 + /home/dae/.rvm/wrappers/ruby-2.3.0/ruby /home/dae/cafe_grader/judge/scripts/grader grading queue > /home/dae/grading.log &
6 + fi
Show file before | Show file after | Raw diff | Download diff | Show whitespace Increase context Show commentsHide comments
@@ -100,96 +100,102
100 100 end while true
101 101 end
102 102
103 103 def create_dir_if_not_exists(dir)
104 104 if ! FileTest.exists? dir
105 105 FileUtils.mkdir(dir)
106 106 end
107 107 end
108 108
109 109 def import_problem(ev_dir, problem, testcase_dir, num_testruns, raw_prefix, check_script, options)
110 110 testrun_info = build_testrun_info_from_dir(num_testruns, testcase_dir, raw_prefix)
111 111
112 112 if !(FileTest.exists? ev_dir)
113 113 puts "Testdata dir (#{ev_dir}) not found."
114 114 return
115 115 end
116 116
117 117 problem_dir = "#{ev_dir}/#{problem}"
118 118
119 119 # start working
120 120 puts "creating directories"
121 121
122 122 create_dir_if_not_exists("#{problem_dir}")
123 123 create_dir_if_not_exists("#{problem_dir}/script")
124 124 create_dir_if_not_exists("#{problem_dir}/test_cases")
125 125
126 126 puts "copying testcases"
127 127
128 128 tr_num = 0
129 129
130 130 num_testcases = 0
131 131
132 132 testrun_info.each do |testrun|
133 133 tr_num += 1
134 134 puts "testrun: #{tr_num}"
135 135
136 136 testrun.each do |testcase_info|
137 137 testcase_num, testcase_fname = testcase_info
138 138
139 139 puts "copy #{testcase_fname} to #{testcase_num}"
140 140
141 141 create_dir_if_not_exists("#{problem_dir}/test_cases/#{testcase_num}")
142 142 copy_testcase("#{testcase_dir}",testcase_fname,"#{problem_dir}/test_cases/#{testcase_num}",testcase_num)
143 143
144 144 num_testcases += 1
145 145 end
146 146 end
147 147
148 + #also include any .txt files
149 + Dir.glob("#{testcase_dir}/*.txt") do |file|
150 + puts "copy data file #{file}"
151 + FileUtils.cp(file,"#{problem_dir}")
152 + end
153 +
148 154 # generating all_tests.cfg
149 155 puts "generating testcase config file"
150 156
151 157 template = File.open(SCRIPT_DIR + "/templates/all_tests.cfg.erb").read
152 158 all_test_cfg = ERB.new(template)
153 159
154 160 cfg_file = File.open("#{problem_dir}/test_cases/all_tests.cfg","w")
155 161 cfg_file.puts all_test_cfg.result binding
156 162 cfg_file.close
157 163
158 164 # copy check script
159 165 if res = /^wrapper:(.*)$/.match(check_script)
160 166 # wrapper script
161 167 check_script_fname = res[1]
162 168 script_name = File.basename(check_script_fname)
163 169 check_wrapper_template = File.open(SCRIPT_DIR + "/templates/check_wrapper").read
164 170 check_wrapper = ERB.new(check_wrapper_template)
165 171
166 172 check_file = File.open("#{problem_dir}/script/check","w")
167 173 check_file.puts check_wrapper.result binding
168 174 check_file.close
169 175
170 176 File.chmod(0755,"#{problem_dir}/script/check")
171 177
172 178 FileUtils.cp("#{check_script_fname}", "#{problem_dir}/script/#{script_name}")
173 179 else
174 180 if File.exists?(SCRIPT_DIR + "/templates/check.#{check_script}")
175 181 check_script_fname = SCRIPT_DIR + "/templates/check.#{check_script}"
176 182 else
177 183 check_script_fname = check_script
178 184 end
179 185 FileUtils.cp("#{check_script_fname}", "#{problem_dir}/script/check", :preserve => true)
180 186 end
181 187
182 188 # generating test_request directory
183 189 puts "generating test_request template"
184 190 FileUtils.mkdir_p("#{ev_dir}/test_request/#{problem}/script")
185 191 FileUtils.mkdir_p("#{ev_dir}/test_request/#{problem}/test_cases/1")
186 192
187 193 template = File.open(SCRIPT_DIR + "/templates/test_request_all_tests.cfg.erb").read
188 194 test_request_all_test_cfg = ERB.new(template)
189 195
190 196 cfg_file = File.open("#{ev_dir}/test_request/#{problem}/test_cases/all_tests.cfg","w")
191 197 cfg_file.puts test_request_all_test_cfg.result
192 198 cfg_file.close
193 199
194 200 FileUtils.cp("#{SCRIPT_DIR}/templates/check_empty",
195 201 "#{ev_dir}/test_request/#{problem}/script/check")
Show file before | Show file after | Raw diff | Download diff | Show whitespace Increase context Show commentsHide comments
@@ -1,85 +1,85
1 1 #!/usr/bin/env ruby
2 2
3 3 def config
4 4 Grader::Configuration.get_instance
5 5 end
6 6
7 7 def display_manual
8 8 puts <<USAGE
9 9 load_testcases
10 10 using: load_testcases [problem_name ...]
11 11 problem_name are list of "short name" of the problems
12 12
13 13 options:
14 14 --dry-run do nothing, just simulate the run
15 15 --all import all problem. This might take several minutes
16 16
17 17 USAGE
18 18 end
19 19
20 20 def process_options_and_stop_file
21 21
22 22 # Process 'help' option
23 - if (ARGV.length==1) and (/help/.match(ARGV[0]))
23 + if (ARGV.length == 0) or ((ARGV.length==1) and (/help/.match(ARGV[0])))
24 24 display_manual
25 25 exit(0)
26 26 end
27 27
28 28 #default options
29 29 options = {
30 30 :dry_run => false,
31 31 }
32 32
33 33 options[:dry_run] = (ARGV.delete('--dry') != nil)
34 34 options[:all] = (ARGV.delete('--all') != nil)
35 35
36 36 return options
37 37 end
38 38
39 39 def process_problem(prob,dry_run = false)
40 40 prob.testcases.destroy_all
41 41 testcases_root = File.expand_path(GRADER_ROOT+"/../ev/#{prob.name}/test_cases/")
42 42 num = 1
43 43 puts "Processing problem #{prob.name}"
44 44 loop do
45 45 file_root = testcases_root + "/#{num}/"
46 46 puts " checking file #{file_root}"
47 47 break unless File.exists? file_root
48 48 input = File.read(file_root + "/input-#{num}.txt")
49 49 answer = File.read(file_root + "/answer-#{num}.txt")
50 50 puts " got test case ##{num} of size #{input.size} and #{answer.size}"
51 51
52 52 #THIS IS JUST A PLACE HOLDER
53 53 group = num #this is wrong!!! fix it!!
54 54 score = 10
55 55 #BEWARE
56 56
57 57 prob.testcases.create(input: input,sol: answer, num: num, score:score,group: group) unless dry_run
58 58 num += 1
59 59 end
60 60 end
61 61
62 62 #########################################
63 63 # main program
64 64 #########################################
65 65
66 66 options = process_options_and_stop_file
67 67
68 68 # load grader environment
69 69 GRADER_ENV = 'grading'
70 70 require File.join(File.dirname(__FILE__),'config/environment')
71 71
72 72 # boot rails, to be able to use the active record
73 73 RAILS_ENV = config.rails_env
74 74 require RAILS_ROOT + '/config/environment'
75 75
76 76 if options[:all]
77 - Problem.all.each { |prob| process_problem(prob,options[:all]) }
77 + Problem.all.each { |prob| process_problem(prob,options[:dry_run]) }
78 78 else
79 79 ARGV.each do |name|
80 80 prob = Problem.find_by(name: name)
81 81 process_problem(prob,options[:dry_run]) if prob
82 82 puts "Cannot find the problem #{name}" unless prob
83 83 end
84 84 end
85 85
Show file before | Show file after | Raw diff | Download diff | Show whitespace Increase context Show commentsHide comments
@@ -1,100 +1,100
1 1 #!/usr/bin/env ruby
2 2
3 3 ENVIRONMENT_DIRS = ['ev', 'ev-exam']
4 4
5 5 def config
6 6 Grader::Configuration.get_instance
7 7 end
8 8
9 9 def rename_problem(old_problem_name, new_problem_name)
10 10
11 - if valid_problem_name(new_problem_name)
11 + unless valid_problem_name(new_problem_name)
12 12 puts "Bad new problem name: #{new_problem_name}"
13 13 return
14 14 end
15 15
16 16 problem = Problem.find_by_name(old_problem_name)
17 17 if problem==nil
18 18 puts "Problem #{old_problem_name} does not exist."
19 19 return
20 20 end
21 21
22 22 puts "Problem: #{old_problem_name} -> #{new_problem_name}"
23 23
24 24 ENVIRONMENT_DIRS.each do |dir|
25 25 problem_dir = File.join(GRADER_ROOT,'..',dir,old_problem_name)
26 26 new_problem_dir = File.join(GRADER_ROOT,'..',dir,new_problem_name)
27 27
28 28 if FileTest.exists? problem_dir
29 29 puts "Moving #{problem_dir} to #{new_problem_dir}."
30 30 File.rename(problem_dir, new_problem_dir)
31 31
32 32 tr_problem_dir = File.join(GRADER_ROOT,'..',dir,
33 33 'test_request',old_problem_name)
34 34 new_tr_problem_dir = File.join(GRADER_ROOT,'..',dir,
35 35 'test_request',new_problem_name)
36 36 File.rename(tr_problem_dir, new_tr_problem_dir)
37 37 end
38 38 end
39 39
40 40 problem.name = new_problem_name
41 41 problem.save
42 42 end
43 43
44 44 def usage
45 45 puts <<USAGE
46 46 Usage:
47 47 rename_problem [old_name] [new_name]
48 48 or
49 49 rename_problem -f [filename]
50 50
51 51 When using with -f, that file should contain, for each line, the old
52 52 problem name and its new name.
53 53
54 54 This script should be called at the judge root dir where dirs 'ev' and
55 55 'ev-exam' are.
56 56 USAGE
57 57 end
58 58
59 59 def valid_problem_name(name)
60 60 if name.length==0:
61 61 return false
62 62 else
63 - return !(/^[a-zA-Z0-9_\-]+$/ === name)
63 + return (/^[a-zA-Z0-9_\-]+$/ === name)
64 64 end
65 65 end
66 66
67 67 if (ARGV.length!=2)
68 68 usage
69 69 exit(0)
70 70 end
71 71
72 72 if ARGV[0]=='-f' and !FileTest.exists?(ARGV[1])
73 73 puts "File #{ARGV[1]} does not exist."
74 74 usage
75 75 exit(0)
76 76 end
77 77
78 78 # load grader environment
79 79 GRADER_ENV = 'grading'
80 80 require File.join(File.dirname(__FILE__),'config/environment')
81 81
82 82 # boot rails, to be able to rename the problem
83 83 RAILS_ENV = config.rails_env
84 84 require RAILS_ROOT + '/config/environment'
85 85
86 86 if ARGV[0]!='-f'
87 87 old_problem_name = ARGV[0]
88 88 new_problem_name = ARGV[1]
89 89
90 90 rename_problem(old_problem_name, new_problem_name)
91 91 else
92 92 lines = IO.readlines(ARGV[1])
93 93 lines.each do |line|
94 94 items = line.split
95 95 if items.length==2
96 96 old_name, new_name = items
97 97 rename_problem(old_name, new_name)
98 98 end
99 99 end
100 100 end
Show file before | Show file after | Raw diff | Download diff | Show whitespace Increase context Show commentsHide comments
@@ -95,86 +95,92
95 95 }
96 96 begin
97 97 Dir.chdir sandbox_dir
98 98 rescue
99 99 log "ERROR: Cannot change directory to #{sandbox_dir}."
100 100 exit(127)
101 101 end
102 102 execute("#{problem_home}/script/compile #{language} #{source_file}", "Compilation error!")
103 103 compile_message = open("compiler_message").read
104 104 compile_message.strip!
105 105 call_and_log("Cannot move the compiler message to #{test_result_dir}.") {
106 106 FileUtils.mv("compiler_message", test_result_dir)
107 107 }
108 108 if !FileTest.exist?("a.out")
109 109 log "Cannot compile the source code. See message in #{test_result_dir}/compile_message"
110 110 exit(127)
111 111 else
112 112 call_and_log("Cannot move the compiled program to #{test_result_dir}") {
113 113 FileUtils.mv("a.out",test_result_dir)
114 114 if language == "java" then Dir["*.class"].each { |file| FileUtils.mv(file,test_result_dir)} end
115 115 if language == "python" then Dir["*.pyc"].each { |file| FileUtils.mv(file,test_result_dir)} end
116 116 }
117 117 FileUtils.rm_rf("#{sandbox_dir}/.")
118 118 end
119 119
120 120 require "#{problem_home}/script/test_dsl.rb"
121 121 load "#{problem_home}/test_cases/all_tests.cfg"
122 122 problem = Problem.get_instance
123 123
124 124 if problem.well_formed? == false
125 125 log "The problem specification is not well formed."
126 126 exit(127)
127 127 end
128 128
129 129 # Doing the testing.
130 130 (1..(problem.num_tests)).each do |test_num|
131 131
132 132 $stdout.print "[#{test_num}]"
133 133 $stdout.flush
134 134
135 135 log "Test number: #{test_num}"
136 136
137 137 call_and_log("Cannot copy the compiled program into #{sandbox_dir}") {
138 138 FileUtils.cp("#{test_result_dir}/a.out", sandbox_dir, :preserve => true)
139 139 if language == "java" then Dir["#{test_result_dir}/*.class"].each { |file| FileUtils.cp(file,sandbox_dir)} end
140 140 if language == "python" then Dir["#{test_result_dir}/*.pyc"].each { |file| FileUtils.cp(file,sandbox_dir)} end
141 141 }
142 142
143 + #additionally copy any extra .txt file
144 + data_files = Dir[problem_home + '/*.txt']
145 + data_files.each do |file|
146 + FileUtils.cp(file,sandbox_dir)
147 + end
148 +
143 149 begin
144 150 execute("#{problem_home}/script/run #{language} #{test_num} ", "Error occured during execution of the run script")
145 151 rescue
146 152 # do nothing
147 153 end
148 154
149 155 call_and_log("Cannot create directory #{test_result_dir}/#{test_num}") {
150 156 FileUtils.mkdir "#{test_result_dir}/#{test_num}"
151 157 }
152 158 call_and_log("Cannot copy the result file into #{test_result_dir}/#{test_num}") {
153 159 FileUtils.mv "#{sandbox_dir}/result", "#{test_result_dir}/#{test_num}"
154 160 }
155 161 call_and_log("Cannot copy the comment file into #{test_result_dir}/#{test_num}") {
156 162 FileUtils.mv "#{sandbox_dir}/comment", "#{test_result_dir}/#{test_num}"
157 163 }
158 164 call_and_log("Cannot copy the output file into #{test_result_dir}/#{test_num}") {
159 165 FileUtils.mv "#{sandbox_dir}/output.txt", "#{test_result_dir}/#{test_num}"
160 166 }
161 167 call_and_log("Cannot clear #{sandbox_dir}") {
162 168 FileUtils.rm_rf(Dir.glob("#{sandbox_dir}/*"), :secure => true)
163 169 }
164 170 end
165 171
166 172 $stdout.print "[done]\n"
167 173
168 174 # Grade
169 175 log
170 176 log "Grading..."
171 177 begin
172 178 Dir.chdir test_result_dir
173 179 rescue
174 180 log "ERROR: Cannot change directory to #{test_result_dir}."
175 181 exit(127)
176 182 end
177 183 execute("#{problem_home}/script/grade", "An error occured during grading!")
178 184
179 185 log
180 186 log "All done!"
Show file before | Show file after | Raw diff | Download diff | Show whitespace Increase context Show commentsHide comments
@@ -2,65 +2,74
2 2
3 3 problem_home = ENV['PROBLEM_HOME']
4 4 require "#{problem_home}/script/test_dsl.rb"
5 5
6 6 if ARGV.length < 2
7 7 puts "Usage: check <language> <test-number> [<output-file>]"
8 8 exit(0)
9 9 end
10 10
11 11 language = ARGV[0]
12 12 test_num = ARGV[1].to_i
13 13 if ARGV.length >= 3
14 14 output_file_name = ARGV[2]
15 15 else
16 16 output_file_name = "output.txt"
17 17 end
18 18
19 19 load "#{problem_home}/test_cases/all_tests.cfg"
20 20 problem = Problem.get_instance
21 21
22 22 output_file = File.new(output_file_name, "r")
23 23 answer_file = File.new("#{problem_home}/test_cases/#{test_num}/answer-#{test_num}.txt")
24 24 result_file = File.new("check_result", "w")
25 25
26 26 output_file_content = output_file.read
27 27 answer_file_content = answer_file.read
28 28
29 29 report_correct = lambda {
30 30 result_file.write "Correct\n"
31 31 result_file.write problem.get_score(test_num)
32 32 result_file.write "\n"
33 33 result_file.close
34 34 exit(0)
35 35 }
36 36
37 37 report_wrong = lambda {
38 38 result_file.write "Incorrect\n"
39 39 result_file.write "0\n"
40 40 result_file.close
41 41 exit(0)
42 42 }
43 43
44 44 ##################
45 45 # Your code here #
46 46 ##################
47 47
48 48 ########### THIS IS FOR CHECKING FLOAT with EPSILON error ##########
49 49
50 +
51 + def is_float?(fl)
52 + !!Float(fl) rescue false
53 + end
54 +
50 55 EPSILON = 0.000001
51 56
52 57 out_items = output_file_content.split
53 58 ans_items = answer_file_content.split
54 59
55 60 if out_items.length != ans_items.length
56 61 report_wrong.call
57 62 else
58 63 out_items.length.times do |i|
64 + if is_float?(out_items[i]) && is_float?(ans_items[i])
59 65 out_value = out_items[i].to_f
60 66 ans_value = ans_items[i].to_f
61 67 if (out_value - ans_value).abs > EPSILON * [out_value.abs,ans_value.abs].max
62 68 report_wrong.call
63 69 end
70 + else
71 + report_wrong.call if out_items[i] != ans_items[i]
72 + end
64 73 end
65 74 report_correct.call
66 75 end
You need to be logged in to leave comments. Login now