cafe-grader-judge-scripts Commit

cafe-grader-judge-scripts

Commit `r22:a8f2ad67d212` public

parent | child

Description:

Merged new-arch-branch changes 61:73 into the trunk git-svn-id: http://theory.cpe.ku.ac.th/grader/judge/trunk/scripts@74 6386c4cd-e34a-4fa8-8920-d93eb39b512e

Commit status:

[Not Reviewed]

References:

default

Diff options:

| | | |

Comments:

0 Commit comments 0 Inline Comments

Unresolved TODOs:

There are no unresolved TODOs

Add another comment

r22:a8f2ad67d212 - Mon, 03 Mar 2008 18:03:00 - 52 files changed: 851 inserted, 236 deleted

config/env_exam.rb.SAMPLE created 644 ¶ +20

@@ -0,0 +1,20
	1	+
	2	+
	3	+ Grader::Initializer.run do \|config\|
	4	+
	5	+ config.problems_dir = "/home/jittat/grader/ev"
	6	+ config.user_result_dir = "/home/jittat/grader/result"
	7	+
	8	+ config.talkative = true
	9	+
	10	+ config.report_grader = true
	11	+
	12	+ config.report_comment = lambda do \|comment\|
	13	+ if comment.chomp =~ /^P+$/ # all P's
	14	+ 'passed'
	15	+ else
	16	+ 'failed'
	17	+ end
	18	+ end
	19	+
	20	+ end

config/env_grading.rb.SAMPLE created 644 ¶ +16

@@ -0,0 +1,16
	1	+
	2	+
	3	+ Grader::Initializer.run do \|config\|
	4	+
	5	+ config.problems_dir = "/home/jittat/grader/ev"
	6	+ config.user_result_dir = "/home/jittat/grader/result"
	7	+
	8	+ config.talkative = true
	9	+
	10	+ config.report_grader = true
	11	+
	12	+ config.report_comment = lambda do \|comment\|
	13	+ comment.chomp
	14	+ end
	15	+
	16	+ end

config/env_test.rb.SAMPLE created 644 ¶ +25

@@ -0,0 +1,25
	1	+
	2	+
	3	+ Grader::Initializer.run do \|config\|
	4	+
	5	+ config.problems_dir = "/home/jittat/grader/scripts/test/sandbox/ev"
	6	+ config.user_result_dir = "/home/jittat/grader/scripts/test/sandbox/result"
	7	+
	8	+ config.talkative = true
	9	+
	10	+ config.report_grader = false
	11	+
	12	+ config.rails_env = 'test'
	13	+
	14	+ config.report_comment = lambda do \|comment\|
	15	+ comment.chomp
	16	+ end
	17	+
	18	+ class << config
	19	+ attr_accessor :test_data_dir, :test_sandbox_dir
	20	+ end
	21	+
	22	+ config.test_data_dir = "/home/jittat/grader/scripts/test/data"
	23	+ config.test_sandbox_dir = "/home/jittat/grader/scripts/test/sandbox"
	24	+
	25	+ end

config/environment.rb.SAMPLE created 644 ¶ +11

@@ -0,0 +1,11
	1	+
	2	+ # Rails app directory
	3	+ RAILS_ROOT = "/home/jittat/web_grader"
	4	+
	5	+ GRADER_ROOT = "/home/jittat/grader/scripts"
	6	+
	7	+ require File.join(File.dirname(__FILE__),'../lib/boot')
	8	+
	9	+ # load the required environment file
	10	+ require File.dirname(__FILE__) + "/env_#{GRADER_ENV}.rb"
	11	+

lib/boot.rb created 644 ¶ +5

@@ -0,0 +1,5
	1	+
	2	+ require File.join(File.dirname(__FILE__), 'configuration')
	3	+ require File.join(File.dirname(__FILE__), 'initializer')
	4	+ require File.join(File.dirname(__FILE__), 'engine')
	5	+

lib/configuration.rb created 644 ¶ +38

@@ -0,0 +1,38
	1	+
	2	+ module Grader
	3	+
	4	+ class Configuration
	5	+
	6	+ private_class_method :new
	7	+
	8	+ attr_accessor :problems_dir
	9	+ attr_accessor :user_result_dir
	10	+ attr_accessor :talkative
	11	+ attr_accessor :report_grader
	12	+ attr_accessor :grader_hostname
	13	+ attr_accessor :report_comment
	14	+ attr_accessor :rails_env
	15	+
	16	+ @@instance = nil
	17	+
	18	+ def self.get_instance
	19	+ if @@instance==nil
	20	+ @@instance = new
	21	+ end
	22	+ @@instance
	23	+ end
	24	+
	25	+ private
	26	+ def initialize
	27	+ @talkative = false
	28	+ @report_grader = false
	29	+ @grader_hostname = `hostname`.chomp
	30	+
	31	+ @rails_env = 'development'
	32	+
	33	+ @report_comment = lambda { \|comment\| comment }
	34	+ end
	35	+
	36	+ end
	37	+
	38	+ end

lib/engine.rb created 644 ¶ +183

@@ -0,0 +1,183
	1	+
	2	+ module Grader
	3	+
	4	+ class Engine
	5	+
	6	+ def initialize(grader_process=nil)
	7	+ @config = Grader::Configuration.get_instance
	8	+ @grader_process = grader_process
	9	+ end
	10	+
	11	+ def grade(sub)
	12	+ current_dir = `pwd`.chomp
	13	+
	14	+ submission_id = sub.id
	15	+ user = sub.user
	16	+ problem = sub.problem
	17	+
	18	+ # TODO: will have to create real exception for this
	19	+ raise "improper submission" if user==nil or problem==nil
	20	+
	21	+ language = sub.language.name
	22	+ lang_ext = sub.language.ext
	23	+ # FIX THIS
	24	+ talk 'some hack on language'
	25	+ if language == 'cpp'
	26	+ language = 'c++'
	27	+ end
	28	+
	29	+ user_dir = "#{@config.user_result_dir}/#{user.login}"
	30	+ problem_out_dir = "#{user_dir}/#{problem.name}"
	31	+ submission_out_dir = "#{user_dir}/#{problem.name}/#{submission_id}"
	32	+
	33	+ mkdir_if_does_not_exist(user_dir)
	34	+ mkdir_if_does_not_exist(problem_out_dir)
	35	+ mkdir_if_does_not_exist(submission_out_dir)
	36	+
	37	+ problem_home = "#{@config.problems_dir}/#{problem.name}"
	38	+ source_name = "#{problem.name}.#{lang_ext}"
	39	+
	40	+ save_source(sub,submission_out_dir,source_name)
	41	+
	42	+ copy_log = copy_script(problem_home)
	43	+
	44	+ call_judge(problem_home,language,submission_out_dir,source_name)
	45	+ save_result(sub,read_result("#{submission_out_dir}/test-result"))
	46	+
	47	+ clear_script(copy_log,problem_home)
	48	+
	49	+ Dir.chdir(current_dir)
	50	+ end
	51	+
	52	+ def grade_oldest_task
	53	+ task = Task.get_inqueue_and_change_status(Task::STATUS_GRADING)
	54	+ if task!=nil
	55	+ @grader_process.report_active(task) if @grader_process!=nil
	56	+
	57	+ submission = Submission.find(task.submission_id)
	58	+ grade(submission)
	59	+ task.status_complete!
	60	+ end
	61	+ return task
	62	+ end
	63	+
	64	+ def grade_problem(problem)
	65	+ users = User.find(:all)
	66	+ users.each do \|u\|
	67	+ puts "user: #{u.login}"
	68	+ last_sub = Submission.find(:first,
	69	+ :conditions => "user_id = #{u.id} and " +
	70	+ "problem_id = #{prob.id}",
	71	+ :order => 'submitted_at DESC')
	72	+ if last_sub!=nil
	73	+ grade(last_sub)
	74	+ end
	75	+ end
	76	+ end
	77	+
	78	+ protected
	79	+
	80	+ def talk(str)
	81	+ if @config.talkative
	82	+ puts str
	83	+ end
	84	+ end
	85	+
	86	+ def save_source(submission,dir,fname)
	87	+ f = File.open("#{dir}/#{fname}","w")
	88	+ f.write(submission.source)
	89	+ f.close
	90	+ end
	91	+
	92	+ def call_judge(problem_home,language,submission_out_dir,fname)
	93	+ ENV['PROBLEM_HOME'] = problem_home
	94	+
	95	+ talk submission_out_dir
	96	+ Dir.chdir submission_out_dir
	97	+ cmd = "#{problem_home}/script/judge #{language} #{fname}"
	98	+ talk "CMD: #{cmd}"
	99	+ system(cmd)
	100	+ end
	101	+
	102	+ def read_result(test_result_dir)
	103	+ cmp_msg_fname = "#{test_result_dir}/compiler_message"
	104	+ cmp_file = File.open(cmp_msg_fname)
	105	+ cmp_msg = cmp_file.read
	106	+ cmp_file.close
	107	+
	108	+ result_fname = "#{test_result_dir}/result"
	109	+ comment_fname = "#{test_result_dir}/comment"
	110	+ if FileTest.exist?(result_fname)
	111	+ result_file = File.open(result_fname)
	112	+ result = result_file.readline.to_i
	113	+ result_file.close
	114	+
	115	+ comment_file = File.open(comment_fname)
	116	+ comment = comment_file.readline.chomp
	117	+ comment_file.close
	118	+
	119	+ return {:points => result,
	120	+ :comment => comment,
	121	+ :cmp_msg => cmp_msg}
	122	+ else
	123	+ return {:points => 0,
	124	+ :comment => 'compile error',
	125	+ :cmp_msg => cmp_msg}
	126	+ end
	127	+ end
	128	+
	129	+ def save_result(submission,result)
	130	+ problem = submission.problem
	131	+ submission.graded_at = Time.now
	132	+ points = result[:points]
	133	+ submission.points = points
	134	+ comment = @config.report_comment.call(result[:comment])
	135	+ if problem == nil
	136	+ submission.grader_comment = 'PASSED: ' + comment + '(problem is nil)'
	137	+ elsif points == problem.full_score
	138	+ submission.grader_comment = 'PASSED: ' + comment
	139	+ else
	140	+ submission.grader_comment = 'FAILED: ' + comment
	141	+ end
	142	+ submission.compiler_message = result[:cmp_msg]
	143	+ submission.save
	144	+ end
	145	+
	146	+ def get_std_script_dir
	147	+ GRADER_ROOT + '/std-script'
	148	+ end
	149	+
	150	+ def copy_script(problem_home)
	151	+ script_dir = "#{problem_home}/script"
	152	+ std_script_dir = get_std_script_dir
	153	+
	154	+ raise "std-script directory not found" if !FileTest.exist?(std_script_dir)
	155	+
	156	+ scripts = Dir[std_script_dir + '/*']
	157	+
	158	+ copied = []
	159	+
	160	+ scripts.each do \|s\|
	161	+ fname = File.basename(s)
	162	+ if !FileTest.exist?("#{script_dir}/#{fname}")
	163	+ copied << fname
	164	+ system("cp #{s} #{script_dir}")
	165	+ end
	166	+ end
	167	+
	168	+ return copied
	169	+ end
	170	+
	171	+ def clear_script(log,problem_home)
	172	+ log.each do \|s\|
	173	+ system("rm #{problem_home}/script/#{s}")
	174	+ end
	175	+ end
	176	+
	177	+ def mkdir_if_does_not_exist(dirname)
	178	+ Dir.mkdir(dirname) if !FileTest.exist?(dirname)
	179	+ end
	180	+
	181	+ end
	182	+
	183	+ end

lib/initializer.rb created 644 ¶ +13

@@ -0,0 +1,13
	1	+
	2	+ module Grader
	3	+
	4	+ class Initializer
	5	+
	6	+ def self.run(&block)
	7	+ config = Grader::Configuration.get_instance
	8	+ yield config
	9	+ end
	10	+
	11	+ end
	12	+
	13	+ end

test/data/ev/test1/script/check created 755 ¶ +59

@@ -0,0 +1,59
	1	+ #!/usr/bin/ruby
	2	+
	3	+ problem_home = ENV['PROBLEM_HOME']
	4	+ require "#{problem_home}/script/test_dsl.rb"
	5	+
	6	+ if ARGV.length < 2
	7	+ puts "Usage: check <language> <test-number> [<output-file>]"
	8	+ exit(0)
	9	+ end
	10	+
	11	+ language = ARGV[0]
	12	+ test_num = ARGV[1].to_i
	13	+ if ARGV.length >= 3
	14	+ output_file_name = ARGV[2]
	15	+ else
	16	+ output_file_name = "output.txt"
	17	+ end
	18	+
	19	+ load "#{problem_home}/test_cases/all_tests.cfg"
	20	+ problem = Problem.get_instance
	21	+
	22	+ output_file = File.new(output_file_name, "r")
	23	+ answer_file = File.new("#{problem_home}/test_cases/#{test_num}/answer-#{test_num}.txt")
	24	+ result_file = File.new("check_result", "w")
	25	+
	26	+ output_file_content = output_file.read
	27	+ answer_file_content = answer_file.read
	28	+
	29	+ report_correct = lambda {
	30	+ result_file.write "Correct\n"
	31	+ result_file.write problem.get_score(test_num)
	32	+ result_file.write "\n"
	33	+ result_file.close
	34	+ exit(0)
	35	+ }
	36	+
	37	+ report_wrong = lambda {
	38	+ result_file.write "Incorrect\n"
	39	+ result_file.write "0\n"
	40	+ result_file.close
	41	+ exit(0)
	42	+ }
	43	+
	44	+ ##################
	45	+ # Your code here #
	46	+ ##################
	47	+ num_pattern = /^[0-9]*/
	48	+ if (output_file_content =~ num_pattern) == nil
	49	+ report_wrong.call
	50	+ end
	51	+
	52	+ output_i = output_file_content.to_i
	53	+ answer_i = answer_file_content.to_i
	54	+
	55	+ if output_i == answer_i
	56	+ report_correct.call
	57	+ else
	58	+ report_wrong.call
	59	+ end

test/data/ev/test1/test_cases/1/answer-1.txt created 644 ¶ +1

	@@ -0,0 +1,1
				1	+ 2 No newline at end of file

test/data/ev/test1/test_cases/1/input-1.txt created 644 ¶ +1

	@@ -0,0 +1,1
				1	+ 1 1 No newline at end of file

test/data/ev/test1/test_cases/10/answer-10.txt created 644 ¶ +1

	@@ -0,0 +1,1
				1	+ -256 No newline at end of file

test/data/ev/test1/test_cases/10/input-10.txt created 644 ¶ +1

	@@ -0,0 +1,1
				1	+ -128 -128 No newline at end of file

test/data/ev/test1/test_cases/2/answer-2.txt created 644 ¶ +1

	@@ -0,0 +1,1
				1	+ 32 No newline at end of file

test/data/ev/test1/test_cases/2/input-2.txt created 644 ¶ +1

	@@ -0,0 +1,1
				1	+ 20 12 No newline at end of file

test/data/ev/test1/test_cases/3/answer-3.txt created 644 ¶ +1

	@@ -0,0 +1,1
				1	+ 4 No newline at end of file

test/data/ev/test1/test_cases/3/input-3.txt created 644 ¶ +1

	@@ -0,0 +1,1
				1	+ 1 3 No newline at end of file

test/data/ev/test1/test_cases/3/test.cfg created 644 ¶ +1

	@@ -0,0 +1,1
				1	+ mem_limit 32 No newline at end of file

test/data/ev/test1/test_cases/4/answer-4.txt created 644 ¶ +1

	@@ -0,0 +1,1
				1	+ 64 No newline at end of file

test/data/ev/test1/test_cases/4/input-4.txt created 644 ¶ +1

	@@ -0,0 +1,1
				1	+ 32 32 No newline at end of file

test/data/ev/test1/test_cases/5/answer-5.txt created 644 ¶ +1

	@@ -0,0 +1,1
				1	+ -2 No newline at end of file

test/data/ev/test1/test_cases/5/input-5.txt created 644 ¶ +1

	@@ -0,0 +1,1
				1	+ -1 -1 No newline at end of file

test/data/ev/test1/test_cases/6/answer-6.txt created 644 ¶ +1

	@@ -0,0 +1,1
				1	+ -32 No newline at end of file

test/data/ev/test1/test_cases/6/input-6.txt created 644 ¶ +1

	@@ -0,0 +1,1
				1	+ -16 -16 No newline at end of file

test/data/ev/test1/test_cases/7/answer-7.txt created 644 ¶ +1

	@@ -0,0 +1,1
				1	+ 0 No newline at end of file

test/data/ev/test1/test_cases/7/input-7.txt created 644 ¶ +1

	@@ -0,0 +1,1
				1	+ 0 0 No newline at end of file

test/data/ev/test1/test_cases/8/answer-8.txt created 644 ¶ +1

	@@ -0,0 +1,1
				1	+ -1 No newline at end of file

test/data/ev/test1/test_cases/8/input-8.txt created 644 ¶ +1

	@@ -0,0 +1,1
				1	+ 0 -1 No newline at end of file

test/data/ev/test1/test_cases/8/test.cfg created 644 ¶ +1

	@@ -0,0 +1,1
				1	+ mem_limit 64 No newline at end of file

test/data/ev/test1/test_cases/9/answer-9.txt created 644 ¶ +1

	@@ -0,0 +1,1
				1	+ 256 No newline at end of file

test/data/ev/test1/test_cases/9/input-9.txt created 644 ¶ +1

	@@ -0,0 +1,1
				1	+ 128 128 No newline at end of file

test/data/ev/test1/test_cases/all_tests.cfg created 644 ¶ +39

@@ -0,0 +1,39
	1	+ problem do
	2	+ num_tests 10
	3	+ full_score 135
	4	+ time_limit_each 1
	5	+ mem_limit_each 11
	6	+ score_each 10
	7	+
	8	+ run 1 do
	9	+ tests 1, 2
	10	+ scores 10, 20
	11	+ time_limits 1, 2
	12	+ mem_limits 5, 6
	13	+ end
	14	+
	15	+ run 2 do
	16	+ tests 3, 4, 5, 6, 7
	17	+ score_each 10
	18	+ time_limit_each 3
	19	+ mem_limit_each 3
	20	+ end
	21	+
	22	+ run 3 do
	23	+ tests 8, 9, 10
	24	+ end
	25	+
	26	+ test 8 do
	27	+ score 30
	28	+ time_limit 3
	29	+ mem_limit 10
	30	+ end
	31	+
	32	+ test 9 do
	33	+ score 15
	34	+ end
	35	+
	36	+ test 10 do
	37	+ time_limit 1
	38	+ end
	39	+ end

test/data/ev/test2/script/check created 755 ¶ +59

@@ -0,0 +1,59
	1	+ #!/usr/bin/ruby
	2	+
	3	+ problem_home = ENV['PROBLEM_HOME']
	4	+ require "#{problem_home}/script/test_dsl.rb"
	5	+
	6	+ if ARGV.length < 2
	7	+ puts "Usage: check <language> <test-number> [<output-file>]"
	8	+ exit(0)
	9	+ end
	10	+
	11	+ language = ARGV[0]
	12	+ test_num = ARGV[1].to_i
	13	+ if ARGV.length >= 3
	14	+ output_file_name = ARGV[2]
	15	+ else
	16	+ output_file_name = "output.txt"
	17	+ end
	18	+
	19	+ load "#{problem_home}/test_cases/all_tests.cfg"
	20	+ problem = Problem.get_instance
	21	+
	22	+ output_file = File.new(output_file_name, "r")
	23	+ answer_file = File.new("#{problem_home}/test_cases/#{test_num}/answer-#{test_num}.txt")
	24	+ result_file = File.new("check_result", "w")
	25	+
	26	+ output_file_content = output_file.read
	27	+ answer_file_content = answer_file.read
	28	+
	29	+ report_correct = lambda {
	30	+ result_file.write "Correct\n"
	31	+ result_file.write problem.get_score(test_num)
	32	+ result_file.write "\n"
	33	+ result_file.close
	34	+ exit(0)
	35	+ }
	36	+
	37	+ report_wrong = lambda {
	38	+ result_file.write "Incorrect\n"
	39	+ result_file.write "0\n"
	40	+ result_file.close
	41	+ exit(0)
	42	+ }
	43	+
	44	+ ##################
	45	+ # Your code here #
	46	+ ##################
	47	+ num_pattern = /^[0-9]*/
	48	+ if (output_file_content =~ num_pattern) == nil
	49	+ report_wrong.call
	50	+ end
	51	+
	52	+ output_i = output_file_content.to_i
	53	+ answer_i = answer_file_content.to_i
	54	+
	55	+ if output_i == answer_i
	56	+ report_correct.call
	57	+ else
	58	+ report_wrong.call
	59	+ end

test/data/ev/test2/test_cases/1/answer-1.txt created 644 ¶ +2

@@ -0,0 +1,2
	1	+ 2
	2	+

test/data/ev/test2/test_cases/1/input-1.txt created 644 ¶ +2

@@ -0,0 +1,2
	1	+ 1 1
	2	+

test/data/ev/test2/test_cases/2/answer-2.txt created 644 ¶ +2

@@ -0,0 +1,2
	1	+ 2
	2	+

test/data/ev/test2/test_cases/2/input-2.txt created 644 ¶ +2

@@ -0,0 +1,2
	1	+ 1 1
	2	+

test/data/ev/test2/test_cases/all_tests.cfg created 644 ¶ +20

@@ -0,0 +1,20
	1	+ problem do
	2	+ num_tests 2
	3	+ full_score 10
	4	+ time_limit_each 1
	5	+ mem_limit_each 16
	6	+ score_each 10
	7	+
	8	+ run 1 do
	9	+ tests 1
	10	+ end
	11	+
	12	+ test 2 do
	13	+ time_limit 2
	14	+ end
	15	+
	16	+ run 2 do
	17	+ tests 2
	18	+ end
	19	+
	20	+ end

test/data/test1_compile_error.c created 644 ¶ +10

@@ -0,0 +1,10
	1	+ #include <stdio.h>
	2	+
	3	+ int main()
	4	+ {
	5	+ int a,
	6	+ scanf("%d %d",&a,&b);
	7	+ printf("%d\n",a+b);
	8	+ return 0;
	9	+ }
	10	+

test/data/test1_correct.c created 644 ¶ +10

@@ -0,0 +1,10
	1	+ #include <stdio.h>
	2	+
	3	+ int main()
	4	+ {
	5	+ int a,b;
	6	+ scanf("%d %d",&a,&b);
	7	+ printf("%d\n",a+b);
	8	+ return 0;
	9	+ }
	10	+

test/data/test2_1-5sec.c created 644 ¶ +19

@@ -0,0 +1,19
	1	+ #include <stdio.h>
	2	+
	3	+ int main()
	4	+ {
	5	+ int a,b;
	6	+
	7	+ int c=0;
	8	+
	9	+ scanf("%d %d",&a,&b);
	10	+ printf("%d\n",a+b);
	11	+
	12	+ for(a=0; a<2; a++) {
	13	+ while(c<1150000000) {
	14	+ c++;
	15	+ b+=c;
	16	+ }
	17	+ }
	18	+ }
	19	+

test/data/test2_timeout.c created 644 ¶ +11

@@ -0,0 +1,11
	1	+ #include <stdio.h>
	2	+
	3	+ int main()
	4	+ {
	5	+ int a,b;
	6	+
	7	+ scanf("%d %d",&a,&b);
	8	+ while(1)
	9	+ ;
	10	+ }
	11	+

test/engine_test.rb created 644 ¶ +152

@@ -0,0 +1,152
	1	+ require 'test/unit'
	2	+ require 'rubygems'
	3	+ require 'mocha'
	4	+
	5	+ require File.join(File.dirname(__FILE__),'test_helper')
	6	+
	7	+ class GraderEngineTest < UnitTest.TestCase
	8	+
	9	+ def setup
	10	+ @@lang_c = stub(:name => 'c', :ext => 'c')
	11	+ @@lang_cpp = stub(:name => 'cpp', :ext => 'cpp')
	12	+ @@lang_pas = stub(:name => 'pas', :ext => 'pas')
	13	+
	14	+ @config = Grader::Configuration.get_instance
	15	+
	16	+ @problem_test1 = stub(:id => 1, :name => 'test1', :full_score => 135)
	17	+ @user_user1 = stub(:id => 1, :login => 'user1')
	18	+
	19	+ @engine = Grader::Engine.new
	20	+
	21	+ init_sandbox
	22	+ end
	23	+
	24	+ def teardown
	25	+ end
	26	+
	27	+ def test_grade_oldest_task_with_no_submission
	28	+ Task.expects(:get_inqueue_and_change_status).returns(nil)
	29	+ assert_equal nil, @engine.grade_oldest_task, 'should return nil when there is no task'
	30	+ end
	31	+
	32	+ def test_normal_submission
	33	+ submission = create_test1_submission_mock_from_file("test1_correct.c")
	34	+
	35	+ submission.expects(:graded_at=)
	36	+ submission.expects(:points=).with(135)
	37	+ submission.expects(:grader_comment=).with do \|value\|
	38	+ /^PASSED/.match(value)
	39	+ end
	40	+ submission.expects(:compiler_message=).with('')
	41	+ submission.expects(:save)
	42	+
	43	+ @engine.grade(submission)
	44	+ end
	45	+
	46	+ def test_compile_error_submission
	47	+ submission = create_test1_submission_mock_from_file("test1_compile_error.c")
	48	+
	49	+ submission.expects(:graded_at=)
	50	+ submission.expects(:points=).with(0)
	51	+ submission.expects(:grader_comment=).with('FAILED: compile error')
	52	+ submission.expects(:compiler_message=) do \|value\|
	53	+ /[Ee]rror/.match value
	54	+ end
	55	+ submission.expects(:save)
	56	+
	57	+ @engine.grade(submission)
	58	+ end
	59	+
	60	+ def test_timeout_submission
	61	+ @problem_test2 = stub(:id => 1, :name => 'test2', :full_score => 10)
	62	+ @user_user1 = stub(:id => 1, :login => 'user1')
	63	+
	64	+ submission = create_submission_from_file(1, @user_user1, @problem_test2,
	65	+ "test2_timeout.c")
	66	+
	67	+ submission.expects(:graded_at=)
	68	+ submission.expects(:points=).with(0)
	69	+ submission.expects(:grader_comment=).with do \|value\|
	70	+ /^FAILED: TT$/.match value
	71	+ end
	72	+ submission.expects(:compiler_message=).with('')
	73	+ submission.expects(:save)
	74	+
	75	+ @engine.grade(submission)
	76	+ end
	77	+
	78	+ def test_timeout_submission_running_one_and_a_half_second
	79	+ @problem_test2 = stub(:id => 1, :name => 'test2', :full_score => 10)
	80	+ @user_user1 = stub(:id => 1, :login => 'user1')
	81	+
	82	+ submission = create_submission_from_file(1, @user_user1, @problem_test2,
	83	+ "test2_1-5sec.c")
	84	+
	85	+ submission.expects(:graded_at=)
	86	+ submission.expects(:points=).with(0)
	87	+ submission.expects(:grader_comment=).with do \|value\|
	88	+ /^FAILED: TP$/.match value
	89	+ end
	90	+ submission.expects(:compiler_message=).with('')
	91	+ submission.expects(:save)
	92	+
	93	+ @engine.grade(submission)
	94	+ end
	95	+
	96	+ def test_grade_oldest_task
	97	+ # mock submission
	98	+ submission = create_test1_submission_mock_from_file("test1_correct.c")
	99	+
	100	+ submission.expects(:graded_at=)
	101	+ submission.expects(:points=).with(135)
	102	+ submission.expects(:grader_comment=).with do \|value\|
	103	+ /^PASSED/.match(value)
	104	+ end
	105	+ submission.expects(:compiler_message=).with('')
	106	+ submission.expects(:save)
	107	+
	108	+ # mock task
	109	+ task = stub(:id => 1, :submission_id => submission.id)
	110	+ Task.expects(:get_inqueue_and_change_status).returns(task)
	111	+ task.expects(:status_complete!)
	112	+
	113	+ # mock Submission
	114	+ Submission.expects(:find).with(task.submission_id).returns(submission)
	115	+
	116	+ @engine.grade_oldest_task
	117	+ end
	118	+
	119	+ def test_grade_oldest_task_with_grader_process
	120	+ grader_process = stub
	121	+ grader_process.expects(:report_active)
	122	+
	123	+ @engine = Grader::Engine.new(grader_process)
	124	+
	125	+ test_grade_oldest_task
	126	+ end
	127	+
	128	+ protected
	129	+
	130	+ def clear_sandbox
	131	+ clear_cmd = "rm -rf #{@config.test_sandbox_dir}/*"
	132	+ system(clear_cmd)
	133	+ end
	134	+
	135	+ def init_sandbox
	136	+ clear_sandbox
	137	+ Dir.mkdir @config.user_result_dir
	138	+ cp_cmd = "cp -R #{@config.test_data_dir}/ev #{@config.test_sandbox_dir}"
	139	+ system(cp_cmd)
	140	+ end
	141	+
	142	+ def create_submission_from_file(id, user, problem, source_fname, language = @@lang_c)
	143	+ source = File.open(@config.test_data_dir + "/" + source_fname).read
	144	+ stub(:id => id, :user => user, :problem => problem,
	145	+ :source => source, :language => language)
	146	+ end
	147	+
	148	+ def create_test1_submission_mock_from_file(source_fname)
	149	+ create_submission_from_file(1, @user_user1, @problem_test1, source_fname)
	150	+ end
	151	+
	152	+ end

test/test_helper.rb created 644 ¶ +25

@@ -0,0 +1,25
	1	+
	2	+ # This test helper loads the grader's environment and rails environment
	3	+
	4	+ GRADER_ENV = 'test'
	5	+ require File.join(File.dirname(__FILE__),'../config/environment')
	6	+
	7	+
	8	+ # this shall be removed soon
	9	+ RAILS_ENV = Grader::Configuration.get_instance.rails_env
	10	+ require RAILS_ROOT + '/config/environment'
	11	+
	12	+ # make sure not to access real database!
	13	+ # taken from http://blog.jayfields.com/2006/06/ruby-on-rails-unit-tests.html
	14	+
	15	+ class UnitTest
	16	+ def self.TestCase
	17	+ class << ActiveRecord::Base
	18	+ def connection
	19	+ raise 'You cannot access the database from a unit test'
	20	+ # raise InvalidActionError, 'You cannot access the database from a unit test', caller
	21	+ end
	22	+ end
	23	+ Test::Unit::TestCase
	24	+ end
	25	+ end

grader ¶ +25 -159

@@ -1,134 +1,5
1	#!/usr/bin/ruby	1	#!/usr/bin/ruby
2		2
3	- def talk(str)
4	- if TALKATIVE
5	- puts str
6	- end
7	- end
8	-
9	- def execute(command, error_message="")
10	- if not system(command)
11	- puts "ERROR: #{error_message}"
12	- exit(127)
13	- end
14	- end
15	-
16	- def save_source(submission,dir,fname)
17	- f = File.open("#{dir}/#{fname}","w")
18	- f.write(submission.source)
19	- f.close
20	- end
21	-
22	- def call_judge(problem_home,language,problem_out_dir,fname)
23	- ENV['PROBLEM_HOME'] = problem_home
24	-
25	- puts problem_out_dir
26	- Dir.chdir problem_out_dir
27	- cmd = "#{problem_home}/script/judge #{language} #{fname}"
28	- # puts "CMD: #{cmd}"
29	- system(cmd)
30	- end
31	-
32	- def read_result(test_result_dir)
33	- cmp_msg_fname = "#{test_result_dir}/compiler_message"
34	- cmp_file = File.open(cmp_msg_fname)
35	- cmp_msg = cmp_file.read
36	- cmp_file.close
37	-
38	- result_fname = "#{test_result_dir}/result"
39	- comment_fname = "#{test_result_dir}/comment"
40	- if FileTest.exist?(result_fname)
41	- result_file = File.open(result_fname)
42	- result = result_file.readline.to_i
43	- result_file.close
44	-
45	- comment_file = File.open(comment_fname)
46	- comment = comment_file.readline.chomp
47	- comment_file.close
48	-
49	- return {:points => result,
50	- :comment => comment,
51	- :cmp_msg => cmp_msg}
52	- else
53	- return {:points => 0,
54	- :comment => 'compile error',
55	- :cmp_msg => cmp_msg}
56	- end
57	- end
58	-
59	- def save_result(submission,result)
60	- problem = Problem.find(submission.problem_id)
61	- submission.graded_at = Time.now
62	- submission.points = result[:points]
63	- if submission.points == problem.full_score
64	- submission.grader_comment = 'PASSED: ' + report_comment(result[:comment])
65	- else
66	- submission.grader_comment = 'FAILED: ' + report_comment(result[:comment])
67	- end
68	- submission.compiler_message = result[:cmp_msg]
69	- submission.save
70	- end
71	-
72	- def copy_script(problem_home)
73	- script_dir = "#{problem_home}/script"
74	- std_script_dir = File.dirname(__FILE__) + '/std-script'
75	- scripts = Dir[std_script_dir + '/*']
76	-
77	- copied = []
78	-
79	- scripts.each do \|s\|
80	- fname = File.basename(s)
81	- if !FileTest.exist?("#{script_dir}/#{fname}")
82	- copied << fname
83	- system("cp #{s} #{script_dir}")
84	- end
85	- end
86	-
87	- return copied
88	- end
89	-
90	- def clear_script(log,problem_home)
91	- log.each do \|s\|
92	- system("rm #{problem_home}/script/#{s}")
93	- end
94	- end
95	-
96	- def grade(submission_id)
97	- current_dir = `pwd`.chomp
98	-
99	- sub = Submission.find(submission_id)
100	- user = sub.user
101	- problem = sub.problem
102	-
103	- language = sub.language.name
104	- lang_ext = sub.language.ext
105	- # FIX THIS
106	- talk 'some hack on language'
107	- if language == 'cpp'
108	- language = 'c++'
109	- end
110	-
111	- user_dir = "#{USER_RESULT_DIR}/#{user.login}"
112	- Dir.mkdir(user_dir) if !FileTest.exist?(user_dir)
113	-
114	- problem_out_dir = "#{user_dir}/#{problem.name}/#{submission_id}"
115	- Dir.mkdir(problem_out_dir) if !FileTest.exist?(problem_out_dir)
116	-
117	- problem_home = "#{PROBLEMS_DIR}/#{problem.name}"
118	- source_name = "#{problem.name}.#{lang_ext}"
119	-
120	- save_source(sub,problem_out_dir,source_name)
121	-
122	- copy_log = copy_script(problem_home)
123	-
124	- call_judge(problem_home,language,problem_out_dir,source_name)
125	- save_result(sub,read_result("#{problem_out_dir}/test-result"))
126	-
127	- clear_script(copy_log,problem_home)
128	-
129	- Dir.chdir(current_dir)
130	- end
131	-
132	def stop_grader	3	def stop_grader
133	File.open(File.dirname(__FILE__) + '/stop','w').close	4	File.open(File.dirname(__FILE__) + '/stop','w').close
134	end	5	end
@@ -141,6 +12,16
141	system("rm " + File.dirname(__FILE__) + '/stop')	12	system("rm " + File.dirname(__FILE__) + '/stop')
142	end	13	end
143		14
		15	+ def config
		16	+ Grader::Configuration.get_instance
		17	+ end
		18	+
		19	+ def talk(str)
		20	+ if config.talkative
		21	+ puts str
		22	+ end
		23	+ end
		24	+
144	#########################################	25	#########################################
145	# main program	26	# main program
146	#########################################	27	#########################################
@@ -169,23 +50,27
169	end	50	end
170		51
171	puts "environment: #{GRADER_ENV}"	52	puts "environment: #{GRADER_ENV}"
172	- require File.dirname(__FILE__) ~~+ "~~/environment~~.rb"~~	53	+ require File.join(File.dirname(__FILE__),'config/environment')
173		54
174	#reading rails environment	55	#reading rails environment
175	talk 'Reading rails environment'	56	talk 'Reading rails environment'
176		57
177	- RAILS_ENV = 'development'	58	+ RAILS_ENV = config.rails_env
178	- require RAILS_~~APP_DI~~R + '/config/environment'	59	+ require RAILS_ROOT + '/config/environment'
		60	+
179		61
180	#register grader process	62	#register grader process
181	- if REPORT_GRADER	63	+ if config.report_grader
182	- grader_proc = GraderProcess.register(~~GRADER_IP_ADDRESS~~,	64	+ grader_proc = GraderProcess.register(config.grader_hostname,
183	Process.pid,	65	Process.pid,
184	grader_mode)	66	grader_mode)
185	else	67	else
186	grader_proc = nil	68	grader_proc = nil
187	end	69	end
188		70
		71	+ # create judge engine
		72	+ engine = Grader::Engine.new(grader_proc)
		73	+
189	case grader_mode	74	case grader_mode
190	when "queue"	75	when "queue"
191	talk 'Grader queue'	76	talk 'Grader queue'
@@ -194,19 +79,11
194	if check_stopfile # created by calling grader stop	79	if check_stopfile # created by calling grader stop
195	clear_stopfile	80	clear_stopfile
196	puts "stopped"	81	puts "stopped"
197	- grader_proc.report_inactive if grader_proc!=nil	82	+ break
198	- exit(0)
199	end	83	end
200		84
201	- task = Task.get_inqueue_and_change_status(Task::STATUS_GRADING)	85	+ task = engine.grade_oldest_task
202	- if task~~!=nil~~	86	+ if task==nil
203	- grader_proc.report_active(task) if grader_proc!=nil
204	-
205	- grade(task.submission_id)
206	- task.status_complete
207	- else
208	- #grader_proc.report_active if grader_proc!=nil
209	-
210	sleep(5)	87	sleep(5)
211	end	88	end
212	end	89	end
@@ -218,23 +95,12
218	prob = Problem.find_by_name(ARGV[2])	95	prob = Problem.find_by_name(ARGV[2])
219	if prob==nil	96	if prob==nil
220	puts "cannot find problem: #{ARGV[2]}"	97	puts "cannot find problem: #{ARGV[2]}"
221	- exit(0)	98	+ else
		99	+ engine.grade_problem(prob)
222	end	100	end
223	- users = User.find(:all)	101	+
224	- users.each do \|u\|
225	- puts "user: #{u.login}"
226	- last_sub = Submission.find(:first,
227	- :conditions => "user_id = #{u.id} and " +
228	- "problem_id = #{prob.id}",
229	- :order => 'submitted_at DESC')
230	- if last_sub!=nil
231	- grade(last_sub.id)
232	- end
233	- end
234	end	102	end
235		103
236	# report inactive	104	# report inactive
237	grader_proc.report_inactive if grader_proc!=nil	105	grader_proc.report_inactive if grader_proc!=nil
238		106
239	-
240	-

std-script/compile ¶ +17 -13

               #
               ##############################
+            + talk ()
+            + {
+            +     if [ "$TALKATIVE" != "" ]; then
+            + 	echo "$1"
+            +     fi
+            + }
+            +
               export C_COMPILER=/usr/bin/gcc
               export CPLUSPLUS_COMPILER=/usr/bin/g++
               export PASCAL_COMPILER=/usr/bin/gpc
               if [ $# -ge 1 ]
               then
                   export PROG_LANG=$1
-            -     echo "programming language: ${PROG_LANG}"
+            +     talk "programming language: ${PROG_LANG}"
               fi
               if [ $# -ge 2 ]
               else
                   export SOURCE_FILE=source
               fi
-            - echo "         source file: $SOURCE_FILE"
+            + talk "         source file: $SOURCE_FILE"
               if [ $# -ge 3 ]
               then
               else
                   export OUTPUT_FILE=a.out
               fi
-            - echo "         output file: $OUTPUT_FILE"
+            + talk "         output file: $OUTPUT_FILE"
               if [ $# -eq 4 ]
               then
               else
                   export MESSAGE_FILE=compiler_message
               fi
-            - echo "        message file: $MESSAGE_FILE"
+            + talk "        message file: $MESSAGE_FILE"
-            - echo
               # Remove any remaining output files or message files.
               rm -Rf $OUTPUT_FILE
               # Check if the source file exists before attempt compiling.
               if [ ! -f $SOURCE_FILE ]
               then
-            -     echo "ERROR: The source file does not exist!"
+            +     talk "ERROR: The source file does not exist!"
                   echo "ERROR: The source file did not exist." > $MESSAGE_FILE
                   exit 127
               fi
               then
                   $PASCAL_COMPILER $PASCAL_OPTIONS -o $OUTPUT_FILE $SOURCE_FILE 2>$MESSAGE_FILE
               else
-            -     echo "ERROR: Invalid language specified!"
+            +     talk "ERROR: Invalid language specified!"
                   echo "ERROR: Invalid language specified!" > $MESSAGE_FILE
                   exit 127
               fi
               # Report success or failure.
               if [ -f $OUTPUT_FILE ]
               then
-            -     echo "Compilation was successful!"
+            +     talk "Compilation was successful!"
               else
-            -     echo "ERROR: Something was wrong during the compilation!"
+            +     talk "ERROR: Something was wrong during the compilation!"
-            -     echo "Dumping compiler message:"
+            +     talk "Dumping compiler message:"
-            -     echo
+            +     #cat $MESSAGE_FILE
-            -     cat $MESSAGE_FILE
               fi

std-script/grade ¶ +15 -2

               #!/usr/bin/ruby
+            + def log(str='')
+            +   if ENV['TALKATIVE']!=nil
+            +     puts str
+            +   end
+            +   if ENV['GRADER_LOGGING']!=nil
+            +     log_fname = ENV['GRADER_LOGGING']
+            +     fp = File.open(log_fname,"a")
+            +     fp.puts("grade: #{Time.new.strftime("%H:%M")} #{str}")
+            +     fp.close
+            +   end
+            + end
+            +
               def char_comment(comment)
                 if comment =~ /[iI]ncorrect/
                   '-'
               problem = Problem.get_instance
               if problem.well_formed? == false
-            -   puts "The problem specification is not well formed."
+            +   log "The problem specification is not well formed."
                 exit(127)
               end
               all_score = 0
               all_comment = ''
               (1..(problem.runs.length-1)).each do |k|
+            +   log "grade run #{k}"
                 run = problem.runs[k]
                 run_score = 0
                 run_comment = ''
                 run.tests.each do |test_num|
                   result_file_name = "#{test_num}/result"
                   if not File.exists?(result_file_name)
-            -       puts "Cannot find the file #{test_num}/result!"
+            +       log "Cannot find the file #{test_num}/result!"
                     exit(127)
                   end

std-script/judge ¶ +28 -16

               #!/usr/bin/ruby
+            + def log(str='')
+            +   if ENV['TALKATIVE']!=nil
+            +     puts str
+            +   end
+            +   if ENV['GRADER_LOGGING']!=nil
+            +     log_fname = ENV['GRADER_LOGGING']
+            +     fp = File.open(log_fname,"a")
+            +     fp.puts("judge: #{Time.new.strftime("%H:%M")} #{str}")
+            +     fp.close
+            +   end
+            + end
+            +
               problem_home = ENV['PROBLEM_HOME']
               def execute(command, error_message="")
               language = ARGV[0]
               if language != "c" && language != "c++" && language != "pascal"
-            -   puts "You specified a language that is not supported."
+            +   log "You specified a language that is not supported."
                 exit(127)
               end
               source_file = ARGV[1]
               if File.exist?(source_file) == false
-            -   puts "The source file does not exist."
+            +   log "The source file does not exist."
                 exit(127)
               end
-            - puts "Making test result and sandbox directories..."
+            + log "Making test result and sandbox directories..."
               current_dir = `pwd`
               current_dir.strip!
               else
                 test_result_dir = "#{current_dir}/test-result"
               end
-            - puts "Test result directory: #{test_result_dir}"
+            + log "Test result directory: #{test_result_dir}"
               system("rm -Rf #{test_result_dir}")
               execute("mkdir #{test_result_dir}", "Cannot make directory #{test_result_dir}.")
               else
                 sandbox_dir = "#{current_dir}/sandbox"
               end
-            - puts "Sandbox directory: #{sandbox_dir}"
+            + log "Sandbox directory: #{sandbox_dir}"
               system("rm -Rf #{sandbox_dir}")
               execute("mkdir #{sandbox_dir}", "Cannot make directory #{sandbox_dir}")
               # Compile
-            - puts
+            + log
-            - puts "Compiling..."
+            + log "Compiling..."
               execute("cp #{source_file} #{sandbox_dir}", "Cannot copy the source file to #{sandbox_dir}")
               begin
                 Dir.chdir sandbox_dir
               rescue
-            -   puts "ERROR: Cannot change directory to #{sandbox_dir}."
+            +   log "ERROR: Cannot change directory to #{sandbox_dir}."
                 exit(127)
               end
               execute("#{problem_home}/script/compile #{language} #{source_file}", "Compilation error!")
               compile_message.strip!
               execute("mv compiler_message #{test_result_dir}", "Cannot move the compiler message to #{test_result_dir}.")
               if !FileTest.exist?("a.out")
-            -   puts "Cannot compile the source code. See message in #{test_result_dir}/compile_message"
+            +   log "Cannot compile the source code. See message in #{test_result_dir}/compile_message"
                 exit(127)
               else
                 execute("mv a.out #{test_result_dir}", "Cannot move the compiled program to #{test_result_dir}")
               problem = Problem.get_instance
               if problem.well_formed? == false
-            -   puts "The problem specification is not well formed."
+            +   log "The problem specification is not well formed."
                 exit(127)
               end
               # Doing the testing.
               (1..(problem.num_tests)).each do |test_num|
-            -   puts
+            +   log "Test number: #{test_num}"
                 execute("cp #{test_result_dir}/a.out #{sandbox_dir}", "Cannot copy the compiled program into #{sandbox_dir}")
                 execute("#{problem_home}/script/run #{language} #{test_num}", "Error occured during execution of the run script")
                 execute("mkdir #{test_result_dir}/#{test_num}", "Cannot create directory #{test_result_dir}/#{test_num}")
               end
               # Grade
-            - puts
+            + log
-            - puts "Grading..."
+            + log "Grading..."
               begin
                 Dir.chdir test_result_dir
               rescue
-            -   puts "ERROR: Cannot change directory to #{test_result_dir}."
+            +   log "ERROR: Cannot change directory to #{test_result_dir}."
                 exit(127)
               end
               execute("#{problem_home}/script/grade", "An error occured during grading!")
-            - puts
+            + log
-            - puts "All done!"
+            + log "All done!"

std-script/run ¶ +21 -10

               #!/usr/bin/ruby
+            + def log(str='')
+            +   if ENV['TALKATIVE']!=nil
+            +     puts str
+            +   end
+            +   if ENV['GRADER_LOGGING']!=nil
+            +     log_fname = ENV['GRADER_LOGGING']
+            +     fp = File.open(log_fname,"a")
+            +     fp.puts("run: #{Time.new.strftime("%H:%M")} #{str}")
+            +     fp.close
+            +   end
+            + end
+            +
               if ARGV.length < 2 || ARGV.length > 3
                 puts "Usage: run <language> <test-num> [<program-name>]"
                 exit(127)
               problem = Problem.get_instance
               if problem.well_formed? == false
-            -   puts "The problem specification is not well formed."
+            +   log "The problem specification is not well formed."
                 exit(127)
               end
               # Check if the test number is okay.
               if test_num <= 0 || test_num > problem.num_tests
-            -   puts "You have specified a wrong test number."
+            +   log "You have specified a wrong test number."
                 exit(127)
               end
               # Run the program.
               run_command = "/usr/bin/time -f \"%E\" 2>run_result #{problem_home}/script/box -a 2 -f -t #{time_limit} -m #{mem_limit} -i #{input_file_name} -o output.txt #{program_name}"
-            - puts "Running test #{test_num}..."
+            + log "Running test #{test_num}..."
-            - puts run_command
+            + log run_command
-            - puts
+            + log
               system(run_command)
               # Create the result file.
                 end
                 comment_file.close
-            -   puts
+            +   log "Done!"
-            -   puts "Done!"
                 exit(0)
               }
               if run_result[0][0,2] != "OK"
-            -   puts "There was a runtime error."
+            +   log "There was a runtime error."
                 report.call(run_result[0], 0, "No comment.\n")
               end
               # Run 'check' to evaluate the output.
               #puts "There was no runtime error. Proceed to checking the output."
               check_command = "#{problem_home}/script/check #{language} #{test_num}"
-            - puts "Checking the output..."
+            + log "Checking the output..."
-            - puts check_command
+            + log check_command
               if not system(check_command)
                 exit(127)
               end

env_exam.rb.SAMPLE removed ¶ -17

deleted file

env_grading.rb.SAMPLE removed ¶ -13

deleted file

environment.rb.SAMPLE removed ¶ -6

deleted file

Write
Preview

You need to be logged in to leave comments. Login now

	Site-wide shortcuts
/	Use quick search box
g h	Goto home page
g g	Goto my private gists page
g G	Goto my public gists page
n r	New repository page
n g	New gist page

	Repositories
g s	Goto summary page
g c	Goto changelog page
g f	Goto files page
g F	Goto files page with file search activated
g p	Goto pull requests page
g o	Goto repository settings
g O	Goto repository permissions settings

Sign in to your account

Author

r22:a8f2ad67d212 - Mon, 03 Mar 2008 18:03:00 - 52 files changed: 851 inserted, 236 deleted

Sign in to your account

Commit r22:a8f2ad67d212 public

Author

r22:a8f2ad67d212 - Mon, 03 Mar 2008 18:03:00 - 52 files changed: 851 inserted, 236 deleted

Commit `r22:a8f2ad67d212` public