cafe-grader-judge-scripts Commit

lib/runner.rb created 644 ¶ +54

			@@ -0,0 +1,54
		1	+ #
		2	+ # A runner drives the engine into various tasks.
		3	+ #
		4	+
		5	+ module Grader
		6	+
		7	+ class Runner
		8	+
		9	+ def initialize(engine, grader_process=nil)
		10	+ @engine = engine
		11	+ @grader_process = grader_process
		12	+ end
		13	+
		14	+ def grade_oldest_task
		15	+ task = Task.get_inqueue_and_change_status(Task::STATUS_GRADING)
		16	+ if task!=nil
		17	+ @grader_process.report_active(task) if @grader_process!=nil
		18	+
		19	+ submission = Submission.find(task.submission_id)
		20	+ @engine.grade(submission)
		21	+ task.status_complete!
		22	+ end
		23	+ return task
		24	+ end
		25	+
		26	+ def grade_problem(problem)
		27	+ users = User.find(:all)
		28	+ users.each do \|u\|
		29	+ puts "user: #{u.login}"
		30	+ last_sub = Submission.find(:first,
		31	+ :conditions => "user_id = #{u.id} and " +
		32	+ "problem_id = #{prob.id}",
		33	+ :order => 'submitted_at DESC')
		34	+ if last_sub!=nil
		35	+ @engine.grade(last_sub)
		36	+ end
		37	+ end
		38	+ end
		39	+
		40	+ def grade_oldest_test_request
		41	+ test_request = TestRequest.get_inqueue_and_change_status(Task::STATUS_GRADING)
		42	+ if test_request!=nil
		43	+ @grader_process.report_active(test_request) if @grader_process!=nil
		44	+
		45	+ @engine.grade(test_request)
		46	+ test_request.status_complete!
		47	+ end
		48	+ return test_request
		49	+ end
		50	+
		51	+ end
		52	+
		53	+ end
		54	+

lib/submission_helper.rb created 644 ¶ +95

			@@ -0,0 +1,95
		1	+ module Grader
		2	+
		3	+ class SubmissionRoomMaker
		4	+ def initialize
		5	+ @config = Grader::Configuration.get_instance
		6	+ end
		7	+
		8	+ def produce_grading_room(submission)
		9	+ user = submission.user
		10	+ problem = submission.problem
		11	+ grading_room = "#{@config.user_result_dir}/" +
		12	+ "#{user.login}/#{problem.name}/#{submission.id}"
		13	+
		14	+ FileUtils.mkdir_p(grading_room)
		15	+ grading_room
		16	+ end
		17	+
		18	+ def find_problem_home(submission)
		19	+ problem = submission.problem
		20	+ "#{@config.problems_dir}/#{problem.name}"
		21	+ end
		22	+
		23	+ def save_source(submission,source_name)
		24	+ dir = self.produce_grading_room(submission)
		25	+ f = File.open("#{dir}/#{source_name}","w")
		26	+ f.write(submission.source)
		27	+ f.close
		28	+ end
		29	+
		30	+ def clean_up(submission)
		31	+ end
		32	+ end
		33	+
		34	+ class SubmissionReporter
		35	+ def initialize
		36	+ @config = Grader::Configuration.get_instance
		37	+ end
		38	+
		39	+ def report(sub,test_result_dir)
		40	+ save_result(sub,read_result(test_result_dir))
		41	+ end
		42	+
		43	+ def report_error(sub,msg)
		44	+ save_result(sub,{:points => 0,
		45	+ :comment => "Grading error: #{msg}" })
		46	+ end
		47	+
		48	+ protected
		49	+ def read_result(test_result_dir)
		50	+ cmp_msg_fname = "#{test_result_dir}/compiler_message"
		51	+ cmp_file = File.open(cmp_msg_fname)
		52	+ cmp_msg = cmp_file.read
		53	+ cmp_file.close
		54	+
		55	+ result_fname = "#{test_result_dir}/result"
		56	+ comment_fname = "#{test_result_dir}/comment"
		57	+ if FileTest.exist?(result_fname)
		58	+ result_file = File.open(result_fname)
		59	+ result = result_file.readline.to_i
		60	+ result_file.close
		61	+
		62	+ comment_file = File.open(comment_fname)
		63	+ comment = comment_file.readline.chomp
		64	+ comment_file.close
		65	+
		66	+ return {:points => result,
		67	+ :comment => comment,
		68	+ :cmp_msg => cmp_msg}
		69	+ else
		70	+ return {:points => 0,
		71	+ :comment => 'compile error',
		72	+ :cmp_msg => cmp_msg}
		73	+ end
		74	+ end
		75	+
		76	+ def save_result(submission,result)
		77	+ problem = submission.problem
		78	+ submission.graded_at = Time.now
		79	+ points = result[:points]
		80	+ submission.points = points
		81	+ comment = @config.report_comment(result[:comment])
		82	+ if problem == nil
		83	+ submission.grader_comment = 'PASSED: ' + comment + '(problem is nil)'
		84	+ elsif points == problem.full_score
		85	+ submission.grader_comment = 'PASSED: ' + comment
		86	+ else
		87	+ submission.grader_comment = 'FAILED: ' + comment
		88	+ end
		89	+ submission.compiler_message = result[:cmp_msg] or ''
		90	+ submission.save
		91	+ end
		92	+
		93	+ end
		94	+
		95	+ end

lib/test_request_helper.rb created 644 ¶ +190

			@@ -0,0 +1,190
		1	+ #
		2	+ # This part contains various test_request helpers for interfacing
		3	+ # with Grader::Engine. There are TestRequestRoomMaker and
		4	+ # TestRequestReporter.
		5	+
		6	+ module Grader
		7	+
		8	+ #
		9	+ # A TestRequestRoomMaker is a helper object for Engine
		10	+ # - finds grading room: in user_result_dir/(user)/test_request/ ...
		11	+ # - prepare problem configuration for grading --- basically it copy
		12	+ # all config files, and copy user's input into the testcase
		13	+ # directory. First, it finds the template from problem template
		14	+ # directory; if it can't find a template, it'll use the template
		15	+ # from default template.
		16	+ class TestRequestRoomMaker
		17	+ def initialize
		18	+ @config = Grader::Configuration.get_instance
		19	+ end
		20	+
		21	+ def produce_grading_room(test_request)
		22	+ grading_room = grading_room_dir(test_request)
		23	+ FileUtils.mkdir_p(grading_room)
		24	+ grading_room
		25	+ end
		26	+
		27	+ def find_problem_home(test_request)
		28	+ problem_name = test_request.problem_name
		29	+
		30	+ template_dir = "#{@config.test_request_problem_templates_dir}/" + problem_name
		31	+
		32	+ raise "Test Request: error template not found" if !File.exists?(template_dir)
		33	+
		34	+ problem_home = problem_home_dir(test_request)
		35	+ FileUtils.mkdir_p(problem_home)
		36	+
		37	+ copy_problem_template(template_dir,problem_home)
		38	+ link_input_file(test_request,problem_home)
		39	+
		40	+ problem_home
		41	+ end
		42	+
		43	+ def save_source(test_request,source_name)
		44	+ dir = self.produce_grading_room(test_request)
		45	+ submission = test_request.submission
		46	+ f = File.open("#{dir}/#{source_name}","w")
		47	+ f.write(submission.source)
		48	+ f.close
		49	+ end
		50	+
		51	+ def clean_up(test_request)
		52	+ problem_home = problem_home_dir(test_request)
		53	+ remove_data_files(problem_home)
		54	+ end
		55	+
		56	+ protected
		57	+ def grading_room_dir(test_request)
		58	+ problem_name = test_request.problem_name
		59	+ user = test_request.user
		60	+ "#{@config.user_result_dir}" +
		61	+ "/#{user.login}/test_request" +
		62	+ "/#{problem_name}/#{test_request.id}"
		63	+ end
		64	+
		65	+ def problem_home_dir(test_request)
		66	+ problem_name = test_request.problem_name
		67	+ user = test_request.user
		68	+ "#{@config.user_result_dir}" +
		69	+ "/#{user.login}/test_request/#{problem_name}"
		70	+ end
		71	+
		72	+ def copy_problem_template(template_dir,problem_home)
		73	+ cmd = "cp -R #{template_dir}/* #{problem_home}"
		74	+ system_and_raise_when_fail(cmd,"Test Request: cannot copy problem template")
		75	+ end
		76	+
		77	+ def link_input_file(test_request,problem_home)
		78	+ cmd = "ln -s #{test_request.input_file_name} #{problem_home}/test_cases/1/input-1.txt"
		79	+ system_and_raise_when_fail(cmd,"Test Request: cannot link input file")
		80	+ end
		81	+
		82	+ def remove_data_files(problem_home)
		83	+ if File.exists?("#{problem_home}/test_cases/1/input-1.txt")
		84	+ cmd = "rm #{problem_home}/test_cases/1/*"
		85	+ system_and_raise_when_fail(cmd,"Test Request: cannot remove data files")
		86	+ end
		87	+ end
		88	+
		89	+ def system_and_raise_when_fail(cmd,msg)
		90	+ if !system(cmd)
		91	+ raise msg
		92	+ end
		93	+ end
		94	+
		95	+ end
		96	+
		97	+ class TestRequestReporter
		98	+ def initialize
		99	+ @config = Grader::Configuration.get_instance
		100	+ end
		101	+
		102	+ def report(test_request,test_result_dir)
		103	+ save_result(test_request,read_result(test_result_dir))
		104	+ end
		105	+
		106	+ def report_error(test_request, msg)
		107	+ save_result(test_request, {:running_stat => "#{msg}"})
		108	+ end
		109	+
		110	+ protected
		111	+ def read_result(test_result_dir)
		112	+ # TODO:
		113	+ cmp_msg_fname = "#{test_result_dir}/compiler_message"
		114	+ cmp_file = File.open(cmp_msg_fname)
		115	+ cmp_msg = cmp_file.read
		116	+ cmp_file.close
		117	+
		118	+ result_file_name = "#{test_result_dir}/1/result"
		119	+
		120	+ if File.exists?(result_file_name)
		121	+ output_file_name = "#{test_result_dir}/1/output.txt"
		122	+ results = File.open("#{test_result_dir}/1/result").readlines
		123	+ stat = format_running_stat(results)
		124	+
		125	+ return {
		126	+ :output_file_name => output_file_name,
		127	+ :running_stat => stat,
		128	+ :comment => "",
		129	+ :cmp_msg => cmp_msg}
		130	+ else
		131	+ return {
		132	+ :running_stat => "",
		133	+ :comment => "Compilation error",
		134	+ :cmp_msg => cmp_msg}
		135	+ end
		136	+ end
		137	+
		138	+ def format_running_stat(results)
		139	+ running_time_line = results[-1]
		140	+
		141	+ run_stat = ""
		142	+ if !(/[Cc]orrect/.match(results[0]))
		143	+ run_stat = results[0].chomp
		144	+ end
		145	+
		146	+ if res = /r(.)u(.)s/.match(running_time_line)
		147	+ seconds = (res[1].to_f + res[2].to_f)
		148	+ time_stat = "Time used: #{seconds} sec."
		149	+ else
		150	+ time_stat = "Time used: n/a sec."
		151	+ end
		152	+ return "#{run_stat}#{time_stat}"
		153	+ end
		154	+
		155	+ def save_result(test_request,result)
		156	+ if result[:output_file_name]!=nil
		157	+ test_request.output_file_name = link_output_file(test_request,
		158	+ result[:output_file_name])
		159	+ end
		160	+ test_request.graded_at = Time.now
		161	+ test_request.compiler_message = (result[:cmp_msg] or '')
		162	+ test_request.grader_comment = (result[:comment] or '')
		163	+ test_request.running_stat = (result[:running_stat] or '')
		164	+ test_request.save
		165	+ end
		166	+
		167	+ protected
		168	+ def link_output_file(test_request, fname)
		169	+ target_file_name = random_output_file_name(test_request.user,
		170	+ test_request.problem)
		171	+ FileUtils.mkdir_p(File.dirname(target_file_name))
		172	+ cmd = "ln -s #{fname} #{target_file_name}"
		173	+ if !system(cmd)
		174	+ raise "TestRequestReporter: cannot move output file"
		175	+ end
		176	+ return target_file_name
		177	+ end
		178	+
		179	+ def random_output_file_name(user,problem)
		180	+ problem_name = TestRequest.name_of(problem)
		181	+ begin
		182	+ tmpname = "#{@config.test_request_output_base_dir}" +
		183	+ "/#{user.login}/#{problem_name}/#{rand(10000)}"
		184	+ end while File.exists?(tmpname)
		185	+ tmpname
		186	+ end
		187	+
		188	+ end
		189	+
		190	+ end

templates/all_tests.cfg.erb created 755 ¶ +13

			@@ -0,0 +1,13
		1	+ problem do
		2	+ num_tests <%= num_testcases %>
		3	+ full_score <%= num_testcases*10 %>
		4	+ time_limit_each <%= options[:time_limit] %>
		5	+ mem_limit_each <%= options[:mem_limit] %>
		6	+ score_each 10
		7	+
		8	+ <% 1.upto(num_testcases) do \|i\| %>
		9	+ run <%= i %> do
		10	+ tests <%= i %>
		11	+ end
		12	+ <% end %>
		13	+ end

templates/answer-1.txt created 644 ¶

new file 100644

templates/check_empty created 755 ¶ +47

			@@ -0,0 +1,47
		1	+ #!/usr/bin/ruby
		2	+
		3	+ problem_home = ENV['PROBLEM_HOME']
		4	+ require "#{problem_home}/script/test_dsl.rb"
		5	+
		6	+ if ARGV.length < 2
		7	+ puts "Usage: check <language> <test-number> [<output-file>]"
		8	+ exit(0)
		9	+ end
		10	+
		11	+ language = ARGV[0]
		12	+ test_num = ARGV[1].to_i
		13	+ if ARGV.length >= 3
		14	+ output_file_name = ARGV[2]
		15	+ else
		16	+ output_file_name = "output.txt"
		17	+ end
		18	+
		19	+ load "#{problem_home}/test_cases/all_tests.cfg"
		20	+ problem = Problem.get_instance
		21	+
		22	+ output_file = File.new(output_file_name, "r")
		23	+ answer_file = File.new("#{problem_home}/test_cases/#{test_num}/answer-#{test_num}.txt")
		24	+ result_file = File.new("check_result", "w")
		25	+
		26	+ output_file_content = output_file.read
		27	+ answer_file_content = answer_file.read
		28	+
		29	+ report_correct = lambda {
		30	+ result_file.write "Correct\n"
		31	+ result_file.write problem.get_score(test_num)
		32	+ result_file.write "\n"
		33	+ result_file.close
		34	+ exit(0)
		35	+ }
		36	+
		37	+ report_wrong = lambda {
		38	+ result_file.write "Incorrect\n"
		39	+ result_file.write "0\n"
		40	+ result_file.close
		41	+ exit(0)
		42	+ }
		43	+
		44	+ ##################
		45	+ # Your code here #
		46	+ ##################
		47	+ report_correct.call

templates/test_request_all_tests.cfg.erb created 755 ¶ +12

			@@ -0,0 +1,12
		1	+ problem do
		2	+ num_tests 1
		3	+ full_score 10
		4	+ time_limit_each <%= options[:time_limit] %>
		5	+ mem_limit_each <%= options[:mem_limit] %>
		6	+ score_each 10
		7	+
		8	+ run 1 do
		9	+ tests 1
		10	+ end
		11	+
		12	+ end

test/data/add_nonzero_exit_status.c created 644 ¶ +10

			@@ -0,0 +1,10
		1	+ #include <stdio.h>
		2	+
		3	+ int main()
		4	+ {
		5	+ int a,b;
		6	+ scanf("%d %d",&a,&b);
		7	+ printf("%d\n",a+b);
		8	+ return 10;
		9	+ }
		10	+

test/data/add_too_much_memory_dynamic.c created 644 ¶ +19

			@@ -0,0 +1,19
		1	+ #include <stdio.h>
		2	+ #include <stdlib.h>
		3	+
		4	+ int main()
		5	+ {
		6	+ int a,b;
		7	+ char *huge_array;
		8	+
		9	+ scanf("%d %d",&a,&b);
		10	+
		11	+ huge_array = (char *)malloc(5000000);
		12	+ if(huge_array==NULL)
		13	+ printf("NO!");
		14	+ else
		15	+ printf("%d\n",a+b);
		16	+
		17	+ return 0;
		18	+ }
		19	+

test/data/add_too_much_memory_static.c created 644 ¶ +12

			@@ -0,0 +1,12
		1	+ #include <stdio.h>
		2	+
		3	+ int big_array[2000000];
		4	+
		5	+ int main()
		6	+ {
		7	+ int a,b;
		8	+ scanf("%d %d",&a,&b);
		9	+ printf("%d\n",a+b);
		10	+ return 0;
		11	+ }
		12	+

test/data/ev/test_memory/script/check created 755 ¶ +59

			@@ -0,0 +1,59
		1	+ #!/usr/bin/ruby
		2	+
		3	+ problem_home = ENV['PROBLEM_HOME']
		4	+ require "#{problem_home}/script/test_dsl.rb"
		5	+
		6	+ if ARGV.length < 2
		7	+ puts "Usage: check <language> <test-number> [<output-file>]"
		8	+ exit(0)
		9	+ end
		10	+
		11	+ language = ARGV[0]
		12	+ test_num = ARGV[1].to_i
		13	+ if ARGV.length >= 3
		14	+ output_file_name = ARGV[2]
		15	+ else
		16	+ output_file_name = "output.txt"
		17	+ end
		18	+
		19	+ load "#{problem_home}/test_cases/all_tests.cfg"
		20	+ problem = Problem.get_instance
		21	+
		22	+ output_file = File.new(output_file_name, "r")
		23	+ answer_file = File.new("#{problem_home}/test_cases/#{test_num}/answer-#{test_num}.txt")
		24	+ result_file = File.new("check_result", "w")
		25	+
		26	+ output_file_content = output_file.read
		27	+ answer_file_content = answer_file.read
		28	+
		29	+ report_correct = lambda {
		30	+ result_file.write "Correct\n"
		31	+ result_file.write problem.get_score(test_num)
		32	+ result_file.write "\n"
		33	+ result_file.close
		34	+ exit(0)
		35	+ }
		36	+
		37	+ report_wrong = lambda {
		38	+ result_file.write "Incorrect\n"
		39	+ result_file.write "0\n"
		40	+ result_file.close
		41	+ exit(0)
		42	+ }
		43	+
		44	+ ##################
		45	+ # Your code here #
		46	+ ##################
		47	+ num_pattern = /^[0-9]*/
		48	+ if (output_file_content =~ num_pattern) == nil
		49	+ report_wrong.call
		50	+ end
		51	+
		52	+ output_i = output_file_content.to_i
		53	+ answer_i = answer_file_content.to_i
		54	+
		55	+ if output_i == answer_i
		56	+ report_correct.call
		57	+ else
		58	+ report_wrong.call
		59	+ end

test/data/ev/test_memory/test_cases/1/answer-1.txt created 644 ¶ +2

			@@ -0,0 +1,2
		1	+ 2
		2	+

test/data/ev/test_memory/test_cases/1/input-1.txt created 644 ¶ +2

			@@ -0,0 +1,2
		1	+ 1 1
		2	+

test/data/ev/test_memory/test_cases/2/answer-2.txt created 644 ¶ +2

			@@ -0,0 +1,2
		1	+ 2
		2	+

test/data/ev/test_memory/test_cases/2/input-2.txt created 644 ¶ +2

			@@ -0,0 +1,2
		1	+ 1 1
		2	+

test/data/ev/test_memory/test_cases/all_tests.cfg created 644 ¶ +20

			@@ -0,0 +1,20
		1	+ problem do
		2	+ num_tests 2
		3	+ full_score 20
		4	+ time_limit_each 1
		5	+ mem_limit_each 5
		6	+ score_each 10
		7	+
		8	+ run 1 do
		9	+ tests 1
		10	+ end
		11	+
		12	+ test 2 do
		13	+ mem_limit 10
		14	+ end
		15	+
		16	+ run 2 do
		17	+ tests 2
		18	+ end
		19	+
		20	+ end

test/data/ev/test_normal/script/check created 755 ¶ +59

			@@ -0,0 +1,59
		1	+ #!/usr/bin/ruby
		2	+
		3	+ problem_home = ENV['PROBLEM_HOME']
		4	+ require "#{problem_home}/script/test_dsl.rb"
		5	+
		6	+ if ARGV.length < 2
		7	+ puts "Usage: check <language> <test-number> [<output-file>]"
		8	+ exit(0)
		9	+ end
		10	+
		11	+ language = ARGV[0]
		12	+ test_num = ARGV[1].to_i
		13	+ if ARGV.length >= 3
		14	+ output_file_name = ARGV[2]
		15	+ else
		16	+ output_file_name = "output.txt"
		17	+ end
		18	+
		19	+ load "#{problem_home}/test_cases/all_tests.cfg"
		20	+ problem = Problem.get_instance
		21	+
		22	+ output_file = File.new(output_file_name, "r")
		23	+ answer_file = File.new("#{problem_home}/test_cases/#{test_num}/answer-#{test_num}.txt")
		24	+ result_file = File.new("check_result", "w")
		25	+
		26	+ output_file_content = output_file.read
		27	+ answer_file_content = answer_file.read
		28	+
		29	+ report_correct = lambda {
		30	+ result_file.write "Correct\n"
		31	+ result_file.write problem.get_score(test_num)
		32	+ result_file.write "\n"
		33	+ result_file.close
		34	+ exit(0)
		35	+ }
		36	+
		37	+ report_wrong = lambda {
		38	+ result_file.write "Incorrect\n"
		39	+ result_file.write "0\n"
		40	+ result_file.close
		41	+ exit(0)
		42	+ }
		43	+
		44	+ ##################
		45	+ # Your code here #
		46	+ ##################
		47	+ num_pattern = /^[0-9]*/
		48	+ if (output_file_content =~ num_pattern) == nil
		49	+ report_wrong.call
		50	+ end
		51	+
		52	+ output_i = output_file_content.to_i
		53	+ answer_i = answer_file_content.to_i
		54	+
		55	+ if output_i == answer_i
		56	+ report_correct.call
		57	+ else
		58	+ report_wrong.call
		59	+ end

test/data/ev/test_normal/test_cases/1/answer-1.txt created 644 ¶ +1

			@@ -0,0 +1,1
		1	+ 2 No newline at end of file

test/data/ev/test_normal/test_cases/1/input-1.txt created 644 ¶ +1

			@@ -0,0 +1,1
		1	+ 1 1 No newline at end of file

test/data/ev/test_normal/test_cases/10/answer-10.txt created 644 ¶ +1

			@@ -0,0 +1,1
		1	+ -256 No newline at end of file

test/data/ev/test_normal/test_cases/10/input-10.txt created 644 ¶ +1

			@@ -0,0 +1,1
		1	+ -128 -128 No newline at end of file

test/data/ev/test_normal/test_cases/2/answer-2.txt created 644 ¶ +1

			@@ -0,0 +1,1
		1	+ 32 No newline at end of file

test/data/ev/test_normal/test_cases/2/input-2.txt created 644 ¶ +1

			@@ -0,0 +1,1
		1	+ 20 12 No newline at end of file

test/data/ev/test_normal/test_cases/3/answer-3.txt created 644 ¶ +1

			@@ -0,0 +1,1
		1	+ 4 No newline at end of file

test/data/ev/test_normal/test_cases/3/input-3.txt created 644 ¶ +1

			@@ -0,0 +1,1
		1	+ 1 3 No newline at end of file

test/data/ev/test_normal/test_cases/3/test.cfg created 644 ¶ +1

			@@ -0,0 +1,1
		1	+ mem_limit 32 No newline at end of file

test/data/ev/test_normal/test_cases/4/answer-4.txt created 644 ¶ +1

			@@ -0,0 +1,1
		1	+ 64 No newline at end of file

test/data/ev/test_normal/test_cases/4/input-4.txt created 644 ¶ +1

			@@ -0,0 +1,1
		1	+ 32 32 No newline at end of file

test/data/ev/test_normal/test_cases/5/answer-5.txt created 644 ¶ +1

			@@ -0,0 +1,1
		1	+ -2 No newline at end of file

test/data/ev/test_normal/test_cases/5/input-5.txt created 644 ¶ +1

			@@ -0,0 +1,1
		1	+ -1 -1 No newline at end of file

test/data/ev/test_normal/test_cases/6/answer-6.txt created 644 ¶ +1

			@@ -0,0 +1,1
		1	+ -32 No newline at end of file

test/data/ev/test_normal/test_cases/6/input-6.txt created 644 ¶ +1

			@@ -0,0 +1,1
		1	+ -16 -16 No newline at end of file

test/data/ev/test_normal/test_cases/7/answer-7.txt created 644 ¶ +1

			@@ -0,0 +1,1
		1	+ 0 No newline at end of file

test/data/ev/test_normal/test_cases/7/input-7.txt created 644 ¶ +1

			@@ -0,0 +1,1
		1	+ 0 0 No newline at end of file

test/data/ev/test_normal/test_cases/8/answer-8.txt created 644 ¶ +1

			@@ -0,0 +1,1
		1	+ -1 No newline at end of file

test/data/ev/test_normal/test_cases/8/input-8.txt created 644 ¶ +1

			@@ -0,0 +1,1
		1	+ 0 -1 No newline at end of file

test/data/ev/test_normal/test_cases/8/test.cfg created 644 ¶ +1

			@@ -0,0 +1,1
		1	+ mem_limit 64 No newline at end of file

test/data/ev/test_normal/test_cases/9/answer-9.txt created 644 ¶ +1

			@@ -0,0 +1,1
		1	+ 256 No newline at end of file

test/data/ev/test_normal/test_cases/9/input-9.txt created 644 ¶ +1

			@@ -0,0 +1,1
		1	+ 128 128 No newline at end of file

test/data/ev/test_normal/test_cases/all_tests.cfg created 644 ¶ +39

			@@ -0,0 +1,39
		1	+ problem do
		2	+ num_tests 10
		3	+ full_score 135
		4	+ time_limit_each 1
		5	+ mem_limit_each 11
		6	+ score_each 10
		7	+
		8	+ run 1 do
		9	+ tests 1, 2
		10	+ scores 10, 20
		11	+ time_limits 1, 2
		12	+ mem_limits 5, 6
		13	+ end
		14	+
		15	+ run 2 do
		16	+ tests 3, 4, 5, 6, 7
		17	+ score_each 10
		18	+ time_limit_each 3
		19	+ mem_limit_each 3
		20	+ end
		21	+
		22	+ run 3 do
		23	+ tests 8, 9, 10
		24	+ end
		25	+
		26	+ test 8 do
		27	+ score 30
		28	+ time_limit 3
		29	+ mem_limit 10
		30	+ end
		31	+
		32	+ test 9 do
		33	+ score 15
		34	+ end
		35	+
		36	+ test 10 do
		37	+ time_limit 1
		38	+ end
		39	+ end

test/data/ev/test_timeout/script/check created 755 ¶ +59

			@@ -0,0 +1,59
		1	+ #!/usr/bin/ruby
		2	+
		3	+ problem_home = ENV['PROBLEM_HOME']
		4	+ require "#{problem_home}/script/test_dsl.rb"
		5	+
		6	+ if ARGV.length < 2
		7	+ puts "Usage: check <language> <test-number> [<output-file>]"
		8	+ exit(0)
		9	+ end
		10	+
		11	+ language = ARGV[0]
		12	+ test_num = ARGV[1].to_i
		13	+ if ARGV.length >= 3
		14	+ output_file_name = ARGV[2]
		15	+ else
		16	+ output_file_name = "output.txt"
		17	+ end
		18	+
		19	+ load "#{problem_home}/test_cases/all_tests.cfg"
		20	+ problem = Problem.get_instance
		21	+
		22	+ output_file = File.new(output_file_name, "r")
		23	+ answer_file = File.new("#{problem_home}/test_cases/#{test_num}/answer-#{test_num}.txt")
		24	+ result_file = File.new("check_result", "w")
		25	+
		26	+ output_file_content = output_file.read
		27	+ answer_file_content = answer_file.read
		28	+
		29	+ report_correct = lambda {
		30	+ result_file.write "Correct\n"
		31	+ result_file.write problem.get_score(test_num)
		32	+ result_file.write "\n"
		33	+ result_file.close
		34	+ exit(0)
		35	+ }
		36	+
		37	+ report_wrong = lambda {
		38	+ result_file.write "Incorrect\n"
		39	+ result_file.write "0\n"
		40	+ result_file.close
		41	+ exit(0)
		42	+ }
		43	+
		44	+ ##################
		45	+ # Your code here #
		46	+ ##################
		47	+ num_pattern = /^[0-9]*/
		48	+ if (output_file_content =~ num_pattern) == nil
		49	+ report_wrong.call
		50	+ end
		51	+
		52	+ output_i = output_file_content.to_i
		53	+ answer_i = answer_file_content.to_i
		54	+
		55	+ if output_i == answer_i
		56	+ report_correct.call
		57	+ else
		58	+ report_wrong.call
		59	+ end

test/data/ev/test_timeout/test_cases/1/answer-1.txt created 644 ¶ +2

			@@ -0,0 +1,2
		1	+ 2
		2	+

test/data/ev/test_timeout/test_cases/1/input-1.txt created 644 ¶ +2

			@@ -0,0 +1,2
		1	+ 1 1
		2	+

test/data/ev/test_timeout/test_cases/2/answer-2.txt created 644 ¶ +2

			@@ -0,0 +1,2
		1	+ 2
		2	+

test/data/ev/test_timeout/test_cases/2/input-2.txt created 644 ¶ +2

			@@ -0,0 +1,2
		1	+ 1 1
		2	+

test/data/ev/test_timeout/test_cases/all_tests.cfg created 644 ¶ +20

			@@ -0,0 +1,20
		1	+ problem do
		2	+ num_tests 2
		3	+ full_score 20
		4	+ time_limit_each 1
		5	+ mem_limit_each 16
		6	+ score_each 10
		7	+
		8	+ run 1 do
		9	+ tests 1
		10	+ end
		11	+
		12	+ test 2 do
		13	+ time_limit 2
		14	+ end
		15	+
		16	+ run 2 do
		17	+ tests 2
		18	+ end
		19	+
		20	+ end

test/data/test_request/input/in1.txt created 644 ¶ +2

			@@ -0,0 +1,2
		1	+ 10
		2	+ 20

test/data/test_request/problems/test_normal/script/check created 755 ¶ +47

			@@ -0,0 +1,47
		1	+ #!/usr/bin/ruby
		2	+
		3	+ problem_home = ENV['PROBLEM_HOME']
		4	+ require "#{problem_home}/script/test_dsl.rb"
		5	+
		6	+ if ARGV.length < 2
		7	+ puts "Usage: check <language> <test-number> [<output-file>]"
		8	+ exit(0)
		9	+ end
		10	+
		11	+ language = ARGV[0]
		12	+ test_num = ARGV[1].to_i
		13	+ if ARGV.length >= 3
		14	+ output_file_name = ARGV[2]
		15	+ else
		16	+ output_file_name = "output.txt"
		17	+ end
		18	+
		19	+ load "#{problem_home}/test_cases/all_tests.cfg"
		20	+ problem = Problem.get_instance
		21	+
		22	+ output_file = File.new(output_file_name, "r")
		23	+ answer_file = File.new("#{problem_home}/test_cases/#{test_num}/answer-#{test_num}.txt")
		24	+ result_file = File.new("check_result", "w")
		25	+
		26	+ output_file_content = output_file.read
		27	+ answer_file_content = answer_file.read
		28	+
		29	+ report_correct = lambda {
		30	+ result_file.write "Correct\n"
		31	+ result_file.write problem.get_score(test_num)
		32	+ result_file.write "\n"
		33	+ result_file.close
		34	+ exit(0)
		35	+ }
		36	+
		37	+ report_wrong = lambda {
		38	+ result_file.write "Incorrect\n"
		39	+ result_file.write "0\n"
		40	+ result_file.close
		41	+ exit(0)
		42	+ }
		43	+
		44	+ ##################
		45	+ # Your code here #
		46	+ ##################
		47	+ report_correct.call

test/data/test_request/problems/test_normal/test_cases/1/answer-1.txt created 644 ¶

new file 100644

test/data/test_request/problems/test_normal/test_cases/all_tests.cfg created 644 ¶ +11

			@@ -0,0 +1,11
		1	+ problem do
		2	+ num_tests 1
		3	+ full_score 10
		4	+ time_limit_each 1
		5	+ mem_limit_each 16
		6	+ score_each 10
		7	+
		8	+ run 1 do
		9	+ tests 1
		10	+ end
		11	+ end

test/engine_spec.rb created 644 ¶ +246

			@@ -0,0 +1,246
		1	+ require File.join(File.dirname(__FILE__),'spec_helper')
		2	+ require File.join(File.dirname(__FILE__),'engine_spec_helper')
		3	+
		4	+ describe "A grader engine, when grading submissions" do
		5	+
		6	+ include GraderEngineHelperMethods
		7	+
		8	+ before(:each) do
		9	+ @config = Grader::Configuration.get_instance
		10	+
		11	+ # this test is from Pong
		12	+ @problem_test_normal = stub(Problem,
		13	+ :id => 1, :name => 'test_normal',
		14	+ :full_score => 135)
		15	+ @user_user1 = stub(User,
		16	+ :id => 1, :login => 'user1')
		17	+
		18	+ @engine = Grader::Engine.new
		19	+ init_sandbox
		20	+ end
		21	+
		22	+ it "should grade normal submission" do
		23	+ grader_should(:grade => "test1_correct.c",
		24	+ :on => @problem_test_normal,
		25	+ :and_report => {
		26	+ :score => 135,
		27	+ :comment => /^PASSED/})
		28	+ end
		29	+
		30	+
		31	+ it "should produce error message when submission cannot compile" do
		32	+ grader_should(:grade => "test1_compile_error.c",
		33	+ :on => @problem_test_normal,
		34	+ :and_report => {
		35	+ :score => 0,
		36	+ :comment => 'FAILED: compile error',
		37	+ :compiler_message => /[Ee]rror/})
		38	+ end
		39	+
		40	+ it "should produce timeout error when submission runs forever" do
		41	+ @problem_test_timeout = stub(Problem,
		42	+ :id => 1, :name => 'test_timeout',
		43	+ :full_score => 10)
		44	+ grader_should(:grade => "test2_timeout.c",
		45	+ :on => @problem_test_timeout,
		46	+ :and_report => {
		47	+ :score => 0,
		48	+ :comment => 'FAILED: TT'})
		49	+ end
		50	+
		51	+ it "should produce timeout error correctly when submission runs slower than expected in less than a second" do
		52	+ @problem_test_timeout = stub(Problem,
		53	+ :id => 1, :name => 'test_timeout',
		54	+ :full_score => 20)
		55	+ grader_should(:grade => "test2_1-5sec.c",
		56	+ :on => @problem_test_timeout,
		57	+ :and_report => {
		58	+ :score => 10,
		59	+ :comment => 'FAILED: TP'})
		60	+ end
		61	+
		62	+ it "should produce runtime error when submission uses too much static memory" do
		63	+ @problem_test_memory = stub(Problem,
		64	+ :id => 1, :name => 'test_memory',
		65	+ :full_score => 20)
		66	+ grader_should(:grade => "add_too_much_memory_static.c",
		67	+ :on => @problem_test_memory,
		68	+ :and_report => {
		69	+ :score => 10,
		70	+ :comment => /FAILED: [^P]P/})
		71	+ end
		72	+
		73	+ it "should not allow submission to allocate too much dynamic memory" do
		74	+ @problem_test_memory = stub(Problem,
		75	+ :id => 1, :name => 'test_memory',
		76	+ :full_score => 20)
		77	+ grader_should(:grade => "add_too_much_memory_dynamic.c",
		78	+ :on => @problem_test_memory,
		79	+ :and_report => {
		80	+ :score => 10,
		81	+ :comment => /FAILED: [^P]P/})
		82	+ end
		83	+
		84	+ it "should fail submission with non-zero exit status" do
		85	+ violated("has not been implemented")
		86	+ end
		87	+
		88	+ def grader_should(args)
		89	+ @user1 = stub(User,
		90	+ :id => 1, :login => 'user1')
		91	+ submission =
		92	+ create_submission_from_file(1, @user1, args[:on], args[:grade])
		93	+ submission.should_receive(:graded_at=)
		94	+
		95	+ expected_score = args[:and_report][:score]
		96	+ expected_comment = args[:and_report][:comment]
		97	+ if args[:and_report][:compiler_message]!=nil
		98	+ expected_compiler_message = args[:and_report][:compiler_message]
		99	+ else
		100	+ expected_compiler_message = ''
		101	+ end
		102	+
		103	+ submission.should_receive(:points=).with(expected_score)
		104	+ submission.should_receive(:grader_comment=).with(expected_comment)
		105	+ submission.should_receive(:compiler_message=).with(expected_compiler_message)
		106	+ submission.should_receive(:save)
		107	+
		108	+ @engine.grade(submission)
		109	+ end
		110	+
		111	+ protected
		112	+
		113	+ def create_normal_submission_mock_from_file(source_fname)
		114	+ create_submission_from_file(1, @user_user1, @problem_test_normal, source_fname)
		115	+ end
		116	+
		117	+ end
		118	+
		119	+ describe "A grader engine, when grading test requests" do
		120	+
		121	+ include GraderEngineHelperMethods
		122	+
		123	+ before(:each) do
		124	+ @config = Grader::Configuration.get_instance
		125	+ @engine = Grader::Engine.new(Grader::TestRequestRoomMaker.new,
		126	+ Grader::TestRequestReporter.new)
		127	+ init_sandbox
		128	+ end
		129	+
		130	+ it "should report error if there is no problem template" do
		131	+ problem = stub(Problem,
		132	+ :id => 1, :name => 'nothing')
		133	+ grader_should(:grade => 'test1_correct.c',
		134	+ :on => problem,
		135	+ :with => 'in1.txt',
		136	+ :and_report => {
		137	+ :graded_at= => nil,
		138	+ :compiler_message= => '',
		139	+ :grader_comment= => '',
		140	+ :running_stat= => /template not found/,
		141	+ :save => nil})
		142	+ end
		143	+
		144	+ it "should run test request and produce output file" do
		145	+ problem = stub(Problem,
		146	+ :id => 1, :name => 'test_normal')
		147	+ grader_should(:grade => 'test1_correct.c',
		148	+ :on => problem,
		149	+ :with => 'in1.txt',
		150	+ :and_report => {
		151	+ :graded_at= => nil,
		152	+ :compiler_message= => '',
		153	+ :grader_comment= => '',
		154	+ :running_stat= => /0.0 sec./,
		155	+ :output_file_name= => lambda { \|fname\|
		156	+ File.exists?(fname).should be_true
		157	+ },
		158	+ :save => nil})
		159	+ end
		160	+
		161	+ it "should clean up problem directory after running test request" do
		162	+ problem = stub(Problem,
		163	+ :id => 1, :name => 'test_normal')
		164	+ grader_should(:grade => 'test1_correct.c',
		165	+ :on => problem,
		166	+ :with => 'in1.txt',
		167	+ :and_report => {
		168	+ :graded_at= => nil,
		169	+ :compiler_message= => '',
		170	+ :grader_comment= => '',
		171	+ :running_stat= => nil,
		172	+ :output_file_name= => nil,
		173	+ :save => nil})
		174	+ File.exists?(@config.user_result_dir + "/test_request/test_normal/test_cases/1/input-1.txt").should be_false
		175	+ end
		176	+
		177	+ it "should compile test request with error and report compilation error" do
		178	+ problem = stub(Problem,
		179	+ :id => 1, :name => 'test_normal')
		180	+ grader_should(:grade => 'test1_compile_error.c',
		181	+ :on => problem,
		182	+ :with => 'in1.txt',
		183	+ :and_report => {
		184	+ :graded_at= => nil,
		185	+ :compiler_message= => /.+/,
		186	+ :grader_comment= => /[Cc]ompil.*error/,
		187	+ :running_stat= => '',
		188	+ :save => nil})
		189	+ end
		190	+
		191	+ it "should report exit status" do
		192	+ problem = stub(Problem,
		193	+ :id => 1, :name => 'test_normal')
		194	+ grader_should(:grade => 'add_nonzero_exit_status.c',
		195	+ :on => problem,
		196	+ :with => 'in1.txt',
		197	+ :and_report => {
		198	+ :graded_at= => nil,
		199	+ :compiler_message= => '',
		200	+ :grader_comment= => '',
		201	+ :running_stat= => /[Ee]xit.status.10.*0.0 sec./,
		202	+ :output_file_name= => lambda { \|fname\|
		203	+ File.exists?(fname).should be_true
		204	+ },
		205	+ :save => nil})
		206	+ end
		207	+
		208	+ protected
		209	+ def grader_should(args)
		210	+ @user1 = stub(User,
		211	+ :id => 1, :login => 'user1')
		212	+
		213	+ problem = args[:on]
		214	+ input_file = @config.test_request_input_base_dir + "/" + args[:with]
		215	+
		216	+ submission =
		217	+ create_submission_from_file(1, @user1, args[:on], args[:grade])
		218	+
		219	+ test_request = stub(TestRequest,
		220	+ :id => 1,
		221	+ :user => @user1,
		222	+ :problem => problem,
		223	+ :submission => submission,
		224	+ :input_file_name => input_file,
		225	+ :language => submission.language,
		226	+ :problem_name => problem.name)
		227	+
		228	+ expectations = args[:and_report]
		229	+
		230	+ expectations.each do \|key,val\|
		231	+ if val==nil
		232	+ test_request.should_receive(key)
		233	+ elsif val.class == Proc
		234	+ test_request.should_receive(key) { \|fname\|
		235	+ val.call(fname)
		236	+ }
		237	+ else
		238	+ test_request.should_receive(key).with(val)
		239	+ end
		240	+ end
		241	+
		242	+ @engine.grade(test_request)
		243	+ end
		244	+
		245	+ end
		246	+

test/engine_spec_helper.rb created 644 ¶ +30

			@@ -0,0 +1,30
		1	+ module GraderEngineHelperMethods
		2	+
		3	+ def clear_sandbox
		4	+ config = Grader::Configuration.get_instance
		5	+ clear_cmd = "rm -rf #{config.test_sandbox_dir}/*"
		6	+ system(clear_cmd)
		7	+ end
		8	+
		9	+ def init_sandbox
		10	+ config = Grader::Configuration.get_instance
		11	+ clear_sandbox
		12	+ Dir.mkdir config.user_result_dir
		13	+ cp_cmd = "cp -R #{config.test_data_dir}/ev #{config.test_sandbox_dir}"
		14	+ system(cp_cmd)
		15	+ end
		16	+
		17	+ def create_submission_from_file(id, user, problem,
		18	+ source_fname, language=nil)
		19	+
		20	+ language = stub(Language, :name => 'c', :ext => 'c') if language==nil
		21	+
		22	+ config = Grader::Configuration.get_instance
		23	+ source = File.open(config.test_data_dir + "/" + source_fname).read
		24	+ stub(Submission,
		25	+ :id => id, :user => user, :problem => problem,
		26	+ :source => source, :language => language)
		27	+ end
		28	+
		29	+ end
		30	+

test/runner_spec.rb created 644 ¶ +65

			@@ -0,0 +1,65
		1	+ require File.join(File.dirname(__FILE__),'spec_helper')
		2	+ require File.join(File.dirname(__FILE__),'engine_spec_helper')
		3	+
		4	+ describe "A grader runner, when grade task" do
		5	+
		6	+ include GraderEngineHelperMethods
		7	+
		8	+ before(:each) do
		9	+ @config = Grader::Configuration.get_instance
		10	+ @problem_test_normal = stub(Problem,
		11	+ :id => 1, :name => 'test_normal',
		12	+ :full_score => 135)
		13	+ @user_user1 = stub(User,
		14	+ :id => 1, :login => 'user1')
		15	+
		16	+ @engine = Grader::Engine.new
		17	+ @runner = Grader::Runner.new(@engine)
		18	+ init_sandbox
		19	+ end
		20	+
		21	+ it "should just return nil when there is no submission" do
		22	+ Task.should_receive(:get_inqueue_and_change_status).and_return(nil)
		23	+ @runner.grade_oldest_task.should be_nil
		24	+ end
		25	+
		26	+ it "should grade oldest task in queue" do
		27	+ submission = create_normal_submission_mock_from_file("test1_correct.c")
		28	+
		29	+ submission.should_receive(:graded_at=)
		30	+ submission.should_receive(:points=).with(135)
		31	+ submission.should_receive(:grader_comment=).with(/^PASSED/)
		32	+ submission.should_receive(:compiler_message=).with('')
		33	+ submission.should_receive(:save)
		34	+
		35	+ # mock task
		36	+ task = stub(Task,:id => 1, :submission_id => submission.id)
		37	+ Task.should_receive(:get_inqueue_and_change_status).and_return(task)
		38	+ task.should_receive(:status_complete!)
		39	+
		40	+ # mock Submission
		41	+ Submission.should_receive(:find).
		42	+ with(task.submission_id).
		43	+ and_return(submission)
		44	+
		45	+ @runner.grade_oldest_task
		46	+ end
		47	+
		48	+ # to be converted
		49	+ def test_grade_oldest_task_with_grader_process
		50	+ grader_process = stub
		51	+ grader_process.expects(:report_active)
		52	+
		53	+ @runner = Grader::Runner.new(@engine,grader_process)
		54	+
		55	+ test_grade_oldest_task
		56	+ end
		57	+
		58	+ protected
		59	+
		60	+ def create_normal_submission_mock_from_file(source_fname)
		61	+ create_submission_from_file(1, @user_user1, @problem_test_normal, source_fname)
		62	+ end
		63	+
		64	+ end
		65	+

test/spec_helper.rb created 644 ¶ +25

			@@ -0,0 +1,25
		1	+
		2	+ # This test helper loads the grader's environment and rails environment
		3	+
		4	+ GRADER_ENV = 'test'
		5	+ require File.join(File.dirname(__FILE__),'../config/environment')
		6	+
		7	+
		8	+ # this shall be removed soon
		9	+ RAILS_ENV = Grader::Configuration.get_instance.rails_env
		10	+ require RAILS_ROOT + '/config/environment'
		11	+
		12	+ # make sure not to access real database!
		13	+ # taken from http://blog.jayfields.com/2006/06/ruby-on-rails-unit-tests.html
		14	+
		15	+ class UnitTest
		16	+ def self.TestCase
		17	+ class << ActiveRecord::Base
		18	+ def connection
		19	+ raise 'You cannot access the database from a unit test'
		20	+ # raise InvalidActionError, 'You cannot access the database from a unit test', caller
		21	+ end
		22	+ end
		23	+ Test::Unit::TestCase
		24	+ end
		25	+ end

config/env_exam.rb.SAMPLE ¶ +12 -12

+            -
+            -
+            + #
+            + # See documentation in lib/configuration.rb
+            + #
               Grader::Initializer.run do |config|
-            -   config.problems_dir = "/home/jittat/grader/ev"
-            -   config.user_result_dir = "/home/jittat/grader/result"
+            +   config.problems_dir = GRADER_ROOT + "/../ev"
+            +   config.user_result_dir = GRADER_ROOT + "/../result"
                 config.talkative = true
+            +   config.logging = true
+            +   config.log_dir = GRADER_ROOT + "/../log"
                 config.report_grader = true
-            -   config.report_comment = lambda do |comment|
-            -     if comment.chomp =~ /^P+$/    # all P's
-            -       'passed'
-            -     else
-            -       'failed'
+            +   config.test_request_input_base_dir = RAILS_ROOT + "/data/test_request/input"
+            +   config.test_request_output_base_dir = RAILS_ROOT + "/data/test_request/output"
+            +   config.test_request_problem_templates_dir = config.problems_dir + "/test_request"
+            +
+            +   config.comment_report_style = :short
-                  end
-            -   end
+            -
-            - end

config/env_grading.rb.SAMPLE ¶ +12 -9

+            -
+            -
+            + #
+            + # See documentation in lib/configuration.rb
+            + #
               Grader::Initializer.run do |config|
+            -
-            -   config.problems_dir = "/home/jittat/grader/ev"
-            -   config.user_result_dir = "/home/jittat/grader/result"
+            +   config.problems_dir = GRADER_ROOT + "/../ev"
+            +   config.user_result_dir = GRADER_ROOT + "/../result"
                 config.talkative = true
+            +   config.logging = true
+            +   config.log_dir = GRADER_ROOT + "/../log"
                 config.report_grader = true
-            -   config.report_comment = lambda do |comment|
-            -     comment.chomp
+            +   config.test_request_input_base_dir = RAILS_ROOT + "/data/test_request/input"
+            +   config.test_request_output_base_dir = RAILS_ROOT + "/data/test_request/output"
+            +   config.test_request_problem_templates_dir = config.problems_dir + "/test_request"
+            +
+            +   config.comment_report_style = :full
-                end
+            -
-            - end

config/env_test.rb.SAMPLE ¶ +16 -12

+            -
+            -
+            + #
+            + # See documentation in lib/configuration.rb
+            + #
               Grader::Initializer.run do |config|
+            +   config.problems_dir = GRADER_ROOT + "/test/sandbox/ev"
+            +   config.user_result_dir = GRADER_ROOT + "/test/sandbox/result"
-            -   config.problems_dir = "/home/jittat/grader/scripts/test/sandbox/ev"
-            -   config.user_result_dir = "/home/jittat/grader/scripts/test/sandbox/result"
+            -
-            -   config.talkative = true
+            +   config.talkative = false
                 config.report_grader = false
                 config.rails_env = 'test'
-            -   config.report_comment = lambda do |comment|
-            -     comment.chomp
-            -   end
+            +   config.comment_report_style = :full
+            +   config.test_request_input_base_dir = GRADER_ROOT + "/test/data/test_request/input"
+            +   config.test_request_output_base_dir = GRADER_ROOT + "/test/sandbox/test_request/output"
+            +   config.test_request_problem_templates_dir = GRADER_ROOT + "/test/data/test_request/problems"
+            +
+            +   #
+            +   # These options are for testing
+            +   #
                 class << config
                   attr_accessor :test_data_dir, :test_sandbox_dir
                 end
-            -   config.test_data_dir = "/home/jittat/grader/scripts/test/data"
-            -   config.test_sandbox_dir = "/home/jittat/grader/scripts/test/sandbox"
+            -
+            +   config.test_data_dir = GRADER_ROOT + "/test/data"
+            +   config.test_sandbox_dir = GRADER_ROOT + "/test/sandbox"
               end

config/environment.rb.SAMPLE ¶ +1 -2

+            -
               # Rails app directory
               RAILS_ROOT = "/home/jittat/web_grader"
               GRADER_ROOT = "/home/jittat/grader/scripts"
+            + # This load all required codes
               require File.join(File.dirname(__FILE__),'../lib/boot')
               # load the required environment file
               require File.dirname(__FILE__) + "/env_#{GRADER_ENV}.rb"
+            -

grader ¶ +68 -11

                 Grader::Configuration.get_instance
               end
-            - def talk(str)
+            + def log_file_name
+            +   config.log_dir +
+            +     "/#{GRADER_ENV}_#{config.grader_mode}.#{Process.pid}"
+            + end
+            +
+            + def log(str)
                 if config.talkative
                   puts str
                 end
+            +   if config.logging
+            +     fp = File.open(log_file_name,"a")
+            +     fp.puts("GRADER: #{Time.new.strftime("%H:%M")} #{str}")
+            +     fp.close
+            +   end
+            + end
+            +
+            + def display_manual
+            +   puts <<USAGE
+            + Grader.
+            + using: (1) grader
+            +        (2) grader environment [mode]
+            +        (3) grader stop
+            +        (4) grader --help
+            + (1) call grader with environment = 'exam', mode = 'queue'
+            + (2) possible modes are: 'queue', 'prob', 'test_request'
+            + (3) create stop-file to stop running grader in queue mode
+            + (4) You are here.
+            + USAGE
               end
               #########################################
               # main program
               #########################################
+            + # with --help
+            + if (ARGV.length==1) and (/help/.match(ARGV[0]))
+            +   display_manual
+            +   exit(0)
+            + end
+            +
               # reading environment and options
               if (ARGV.length >= 1) and (ARGV[0]=='stop')
                 stop_grader
               puts "environment: #{GRADER_ENV}"
               require File.join(File.dirname(__FILE__),'config/environment')
+            + # add grader_mode to config
+            + # this is needed because method log needs it.  TODO: clean this up
+            + class << config
+            +   attr_accessor :grader_mode
+            + end
+            + config.grader_mode = grader_mode
+            +
               #reading rails environment
-            - talk 'Reading rails environment'
+            + log 'Reading rails environment'
               RAILS_ENV = config.rails_env
               require RAILS_ROOT + '/config/environment'
+            -
               #register grader process
               if config.report_grader
                 grader_proc = GraderProcess.register(config.grader_hostname,
                 grader_proc = nil
               end
-            - # create judge engine
-            - engine = Grader::Engine.new(grader_proc)
+            + #set loggin environment
+            + ENV['GRADER_LOGGING'] = log_file_name
+            +
+            + #
+            + # MAIN LOOP
+            + #
               case grader_mode
-            - when "queue"
-            -   talk 'Grader queue'
+            + when "queue", "test_request"
+            +   log "Grader: #{grader_mode}"
+            +   if grader_mode=="queue"
+            +     engine = Grader::Engine.new
+            +   else
+            +     engine = Grader::Engine.new(Grader::TestRequestRoomMaker.new,
+            +                                 Grader::TestRequestReporter.new)
+            +   end
+            +
+            +   runner = Grader::Runner.new(engine, grader_proc)
                 while true
                   if check_stopfile    # created by calling grader stop
                     clear_stopfile
-            -       puts "stopped"
+            +       log "stopped (with stop file)"
                     break
                   end
-            -     task = engine.grade_oldest_task
+            +     if grader_mode=="queue"
+            +       task = runner.grade_oldest_task
+            +     else
+            +       task = runner.grade_oldest_test_request
+            +     end
                   if task==nil
-            -       sleep(5)
+            +       sleep(1)
                   end
                 end
               when "prob"
+            +   engine = Grader::Engine.new
+            +   runner = Grader::Runner.new(engine, grader_proc)
                 grader_proc.report_active if grader_proc!=nil
                 if prob==nil
                   puts "cannot find problem: #{ARGV[2]}"
                 else
-            -     engine.grade_problem(prob)
+            +     runner.grade_problem(prob)
                 end
+            + else
+            +   display_manual
+            +   exit(0)
               end
               # report inactive

import_problem ¶ +24 -4

               #  * copy testdata in the old format and create standard testcase config file
               require 'erb'
+            + require 'fileutils'
               def input_filename(dir,i)
                 "#{dir}/input-#{i}.txt"
                 end
               end
-            - GRADER_DIR = File.dirname(__FILE__)
+            + SCRIPT_DIR = File.dirname(__FILE__)
               # print usage
               if ARGV.length < 3
-            -   puts "using: import_problem prob_name importing_testcase_dir num_of_testcase [options]
+            +   puts <<USAGE
+            + using: import_problem prob_name importing_testcase_dir num_of_testcase [options]
                 * creates a directory for a problem in the current directory,
                 * copy testdata in the old format and create standard testcase config file
+            +   * creates a test_request template in the current directory + '/test_request'
                 * options: -t time-limit (in seconds)
-            -              -m memory-limit (in megabytes)"
+            +              -m memory-limit (in megabytes)
+            + USAGE
                 exit(127)
               end
               # start working
               puts "creating directories"
+            +
               system("mkdir #{problem}")
               system("mkdir #{problem}/script")
               system("mkdir #{problem}/test_cases")
               # generating all_tests.cfg
               puts "generating testcase config file"
-            - template = File.open(File.dirname(__FILE__) + "/all_tests.cfg.erb").read
+            + template = File.open(SCRIPT_DIR + "/templates/all_tests.cfg.erb").read
               all_test_cfg = ERB.new(template)
               cfg_file = File.open("#{problem}/test_cases/all_tests.cfg","w")
               cfg_file.puts all_test_cfg.result
               cfg_file.close
+            + # generating test_request directory
+            + puts "generating test_request template"
+            + FileUtils.mkdir_p("test_request/#{problem}/script")
+            + FileUtils.mkdir_p("test_request/#{problem}/test_cases/1")
+            +
+            + template = File.open(SCRIPT_DIR + "/templates/test_request_all_tests.cfg.erb").read
+            + test_request_all_test_cfg = ERB.new(template)
+            +
+            + cfg_file = File.open("test_request/#{problem}/test_cases/all_tests.cfg","w")
+            + cfg_file.puts test_request_all_test_cfg.result
+            + cfg_file.close
+            +
+            + system("cp #{SCRIPT_DIR}/templates/check_empty test_request/#{problem}/script/check")
+            + system("cp #{SCRIPT_DIR}/templates/answer-1.txt test_request/#{problem}/test_cases/1")
+            +
               puts "done"

lib/boot.rb ¶ +5

               require File.join(File.dirname(__FILE__), 'configuration')
               require File.join(File.dirname(__FILE__), 'initializer')
+            +
+            + require File.join(File.dirname(__FILE__), 'submission_helper')
+            + require File.join(File.dirname(__FILE__), 'test_request_helper')
+            +
               require File.join(File.dirname(__FILE__), 'engine')
+            + require File.join(File.dirname(__FILE__), 'runner')

lib/configuration.rb ¶ +51 -7

+            -
-              module Grader
+            +   # This singleton class holds basic configurations for grader.  When
+            +   # running in each mode, grader uses resources from different
+            +   # directories and outputs differently.  Usually the attributes name
+            +   # are descriptive; below we explain more on each attributes.
                 class Configuration
+            +     # Rails' environment: "development", "production"
+            +     attr_accessor :rails_env
-            -     private_class_method :new
+            -
+            +     # Grader looks for problem [prob] in problem_dir/[prob], and store
+            +     # execution results for submission [x] of user [u] in directory
+            +     # user_result_dir/[u]/[x]
                   attr_accessor :problems_dir
                   attr_accessor :user_result_dir
-            -     attr_accessor :talkative
+            +
+            +     # If report_grader=true, the grader would add a row in model
+            +     # GraderProcess.  It would report itself with grader_hostname and
+            +     # process id.
                   attr_accessor :report_grader
                   attr_accessor :grader_hostname
-            -     attr_accessor :report_comment
-            -     attr_accessor :rails_env
+            +
+            +     # If talkative=true, grader would report status to console.  If
+            +     # logging=true, grader would report status to a log file located
+            +     # in log_dir, in a file name mode.options.pid.  TODO: defined
+            +     # log file naming.
+            +     attr_accessor :talkative
+            +     attr_accessor :logging
+            +     attr_accessor :log_dir
+            +
+            +     # These are directories related to the test interface.
+            +     attr_accessor :test_request_input_base_dir
+            +     attr_accessor :test_request_output_base_dir
+            +     attr_accessor :test_request_problem_templates_dir
+            +
+            +     # Comment received from the grading script will be filtered
+            +     # through Configuration#report_comment.  How this method behave
+            +     # depends on this option; right now only two formats, :short and
+            +     # :long
+            +     attr_accessor :comment_report_style
+            +
+            +     def report_comment(comment)
+            +       case comment_report_style
+            +       when :short
+            +         if comment.chomp =~ /^P+$/    # all P's
+            +           'passed'
+            +         else
+            +           'failed'
+            +         end
+            +
+            +       when :full
+            +         comment.chomp
+            +       end
+            +     end
+            +
+            +     # Codes for singleton
+            +     private_class_method :new
                   @@instance = nil
                   private
                   def initialize
                     @talkative = false
+            +       @log_file = nil
                     @report_grader = false
                     @grader_hostname = `hostname`.chomp
                     @rails_env = 'development'
-            -       @report_comment = lambda { |comment| comment }
+            +       @comment_report_style = :full
                   end
                 end

lib/engine.rb ¶ +47 -99

+            + require 'fileutils'
               module Grader
+            +   #
+            +   # A grader engine grades a submission, against anything: a test
+            +   # data, or a user submitted test data.  It uses two helpers objects:
+            +   # room_maker and reporter.
+            +   #
                 class Engine
-            -     def initialize(grader_process=nil)
+            +     attr_writer :room_maker
+            +     attr_writer :reporter
+            +
+            +     def initialize(room_maker=nil, reporter=nil)
                     @config = Grader::Configuration.get_instance
-            -       @grader_process = grader_process
+            +
+            +       @room_maker = room_maker || Grader::SubmissionRoomMaker.new
+            +       @reporter = reporter || Grader::SubmissionReporter.new
                   end
-            -     def grade(sub)
+            +     # takes a submission, asks room_maker to produce grading directories,
+            +     # calls grader scripts, and asks reporter to save the result
+            +     def grade(submission)
                     current_dir = `pwd`.chomp
-            -       submission_id = sub.id
-            -       user = sub.user
-            -       problem = sub.problem
+            +       user = submission.user
+            +       problem = submission.problem
                     # TODO: will have to create real exception for this
-            -       raise "improper submission" if user==nil or problem==nil
+            +       if user==nil or problem == nil
+            +         @reporter.report_error(submission,"Grading error: problem with submission")
+            +         #raise "engine: user or problem is nil"
+            +       end
-            -       language = sub.language.name
-            -       lang_ext = sub.language.ext
+            +       language = submission.language.name
+            +       lang_ext = submission.language.ext
                     # FIX THIS
                     talk 'some hack on language'
                     if language == 'cpp'
                       language = 'c++'
                     end
-            -       user_dir = "#{@config.user_result_dir}/#{user.login}"
-            -       problem_out_dir = "#{user_dir}/#{problem.name}"
-            -       submission_out_dir = "#{user_dir}/#{problem.name}/#{submission_id}"
+            +       # COMMENT: should it be only source.ext?
+            +       if problem!=nil
+            +         source_name = "#{problem.name}.#{lang_ext}"
+            +       else
+            +         source_name = "source.#{lang_ext}"
+            +       end
-            -       mkdir_if_does_not_exist(user_dir)
-            -       mkdir_if_does_not_exist(problem_out_dir)
-            -       mkdir_if_does_not_exist(submission_out_dir)
+            +       begin
+            +         grading_dir = @room_maker.produce_grading_room(submission)
+            +         @room_maker.save_source(submission,source_name)
+            +         problem_home = @room_maker.find_problem_home(submission)
-            -       problem_home = "#{@config.problems_dir}/#{problem.name}"
-            -       source_name = "#{problem.name}.#{lang_ext}"
+            -
-            -       save_source(sub,submission_out_dir,source_name)
+            +         # puts "GRADING DIR: #{grading_dir}"
+            +         # puts "PROBLEM DIR: #{problem_home}"
                     copy_log = copy_script(problem_home)
-            -       call_judge(problem_home,language,submission_out_dir,source_name)
-            -       save_result(sub,read_result("#{submission_out_dir}/test-result"))
+            +         call_judge(problem_home,language,grading_dir,source_name)
+            +
+            +         @reporter.report(submission,"#{grading_dir}/test-result")
                     clear_script(copy_log,problem_home)
-            -       Dir.chdir(current_dir)
-            -     end
+            -
-            -     def grade_oldest_task
-            -       task = Task.get_inqueue_and_change_status(Task::STATUS_GRADING)
-            -       if task!=nil
-            -         @grader_process.report_active(task) if @grader_process!=nil
+            +       rescue RuntimeError => msg
+            +         @reporter.report_error(submission,"Grading error: #{msg}")
-            -         submission = Submission.find(task.submission_id)
-            -         grade(submission)
-            -         task.status_complete!
-            -       end
-            -       return task
-            -     end
+            -
-            -     def grade_problem(problem)
-            -       users = User.find(:all)
-            -       users.each do |u|
-            -         puts "user: #{u.login}"
-            -         last_sub = Submission.find(:first,
-            -                                    :conditions => "user_id = #{u.id} and " +
-            -                                                   "problem_id = #{prob.id}",
-            -                                    :order => 'submitted_at DESC')
-            -         if last_sub!=nil
-            -           grade(last_sub)
-            -         end
+            +       ensure
+            +         @room_maker.clean_up(submission)
+            +         Dir.chdir(current_dir)   # this is really important
                     end
                   end
                     end
                   end
-            -     def save_source(submission,dir,fname)
-            -       f = File.open("#{dir}/#{fname}","w")
-            -       f.write(submission.source)
-            -       f.close
-            -     end
+            -
-            -     def call_judge(problem_home,language,submission_out_dir,fname)
+            +     def call_judge(problem_home,language,grading_dir,fname)
                     ENV['PROBLEM_HOME'] = problem_home
-            -       talk submission_out_dir
-            -       Dir.chdir submission_out_dir
+            +       talk grading_dir
+            +       Dir.chdir grading_dir
                     cmd = "#{problem_home}/script/judge #{language} #{fname}"
                     talk "CMD: #{cmd}"
                     system(cmd)
                   end
-            -     def read_result(test_result_dir)
-            -       cmp_msg_fname = "#{test_result_dir}/compiler_message"
-            -       cmp_file = File.open(cmp_msg_fname)
-            -       cmp_msg = cmp_file.read
-            -       cmp_file.close
+            -
-            -       result_fname = "#{test_result_dir}/result"
-            -       comment_fname = "#{test_result_dir}/comment"
-            -       if FileTest.exist?(result_fname)
-            -         result_file = File.open(result_fname)
-            -         result = result_file.readline.to_i
-            -         result_file.close
+            -
-            -         comment_file = File.open(comment_fname)
-            -         comment = comment_file.readline.chomp
-            -         comment_file.close
+            -
-            -         return {:points => result,
-            -           :comment => comment,
-            -           :cmp_msg => cmp_msg}
-            -       else
-            -         return {:points => 0,
-            -           :comment => 'compile error',
-            -           :cmp_msg => cmp_msg}
-            -       end
-            -     end
+            -
-            -     def save_result(submission,result)
-            -       problem = submission.problem
-            -       submission.graded_at = Time.now
-            -       points = result[:points]
-            -       submission.points = points
-            -       comment = @config.report_comment.call(result[:comment])
-            -       if problem == nil
-            -         submission.grader_comment = 'PASSED: ' + comment + '(problem is nil)'
-            -       elsif points == problem.full_score
-            -         submission.grader_comment = 'PASSED: ' + comment
-            -       else
-            -         submission.grader_comment = 'FAILED: ' + comment
-            -       end
-            -       submission.compiler_message = result[:cmp_msg]
-            -       submission.save
-            -     end
+            -
                   def get_std_script_dir
                     GRADER_ROOT + '/std-script'
                   end

std-script/grade ¶ +1 -1

                 elsif comment =~ /[Tt]ime/
                   'T'
                 else
-            -     '?'
+            +     'x'  # these are run time errors
                 end
               end

std-script/run ¶ +28 -1

                 end
               end
+            + def extract_time(t)
+            +   if (result=/^(.*)r(.*)u(.*)s/.match(t))
+            +     {:real => result[1], :user => result[2], :sys => result[3]}
+            +   else
+            +     raise "Error reading running time"
+            +   end
+            + end
+            +
+            + def compile_box(source,bin)
+            +   system("gcc #{source} -o #{bin}")
+            + end
+            +
               if ARGV.length < 2 || ARGV.length > 3
                 puts "Usage: run <language> <test-num> [<program-name>]"
                 exit(127)
               # Copy the input file.
               #`cp #{problem_home}/test_cases/#{test_num}/#{input_file_name} .`
+            + time_output_format = "%Er%Uu%Ss"
+            +
+            + # check if box is there, if not, compile it!
+            + if !File.exists?("#{problem_home}/script/box")
+            +   log "WARNING: Compiling box: to increase efficiency, it should be compile manually"
+            +   compile_box("#{problem_home}/script/box.c",
+            +               "#{problem_home}/script/box")
+            + end
+            +
               # Run the program.
-            - run_command = "/usr/bin/time -f \"%E\" 2>run_result #{problem_home}/script/box -a 2 -f -t #{time_limit} -m #{mem_limit} -i #{input_file_name} -o output.txt #{program_name}"
+            + run_command = "/usr/bin/time -f \"#{time_output_format}\" 2>run_result #{problem_home}/script/box -a 2 -f -t #{time_limit} -m #{mem_limit} -i #{input_file_name} -o output.txt #{program_name}"
               log "Running test #{test_num}..."
               log run_command
               log
               run_result = run_result_file.readlines
               run_result_file.close
               time_elapsed = run_result[run_result.length-1]
+            + running_time = extract_time(time_elapsed)
               report = lambda{ |status, points, comment|
                 result_file.write status.strip
                 report.call(run_result[0], 0, "No comment.\n")
               end
+            + if running_time[:user].to_f + running_time[:sys].to_f > time_limit
+            +   log "Time limit exceeded."
+            +   report.call("Time limit exceeded", 0, "No comment.\n")
+            + end
+            +
               # Run 'check' to evaluate the output.
               #puts "There was no runtime error. Proceed to checking the output."
               check_command = "#{problem_home}/script/check #{language} #{test_num}"

test/data/test2_1-5sec.c ¶ +2

               #include <stdio.h>
+            + #include <stdlib.h>
               int main()
               {
                     b+=c;
                   }
                 }
+            +   exit(0);
               }

test/engine_test.rb ¶ +2 -2

                 end
                 def test_timeout_submission_running_one_and_a_half_second
-            -     @problem_test2 = stub(:id => 1, :name => 'test2', :full_score => 10)
+            +     @problem_test2 = stub(:id => 1, :name => 'test2', :full_score => 20)
                   @user_user1 = stub(:id => 1, :login => 'user1')
                   submission = create_submission_from_file(1, @user_user1, @problem_test2,
                                                            "test2_1-5sec.c")
                   submission.expects(:graded_at=)
-            -     submission.expects(:points=).with(0)
+            +     submission.expects(:points=).with(10)
                   submission.expects(:grader_comment=).with do |value|
                     /^FAILED: TP$/.match value
                   end

all_tests.cfg.erb removed ¶ -13

deleted file

test/data/ev/test1/script/check removed ¶ -59

deleted file

test/data/ev/test1/test_cases/1/answer-1.txt removed ¶ -1

deleted file

test/data/ev/test1/test_cases/1/input-1.txt removed ¶ -1

deleted file

test/data/ev/test1/test_cases/10/answer-10.txt removed ¶ -1

deleted file

test/data/ev/test1/test_cases/10/input-10.txt removed ¶ -1

deleted file

test/data/ev/test1/test_cases/2/answer-2.txt removed ¶ -1

deleted file

test/data/ev/test1/test_cases/2/input-2.txt removed ¶ -1

deleted file

test/data/ev/test1/test_cases/3/answer-3.txt removed ¶ -1

deleted file

test/data/ev/test1/test_cases/3/input-3.txt removed ¶ -1

deleted file

test/data/ev/test1/test_cases/3/test.cfg removed ¶ -1

deleted file

test/data/ev/test1/test_cases/4/answer-4.txt removed ¶ -1

deleted file

test/data/ev/test1/test_cases/4/input-4.txt removed ¶ -1

deleted file

test/data/ev/test1/test_cases/5/answer-5.txt removed ¶ -1

deleted file

test/data/ev/test1/test_cases/5/input-5.txt removed ¶ -1

deleted file

test/data/ev/test1/test_cases/6/answer-6.txt removed ¶ -1

deleted file

test/data/ev/test1/test_cases/6/input-6.txt removed ¶ -1

deleted file

test/data/ev/test1/test_cases/7/answer-7.txt removed ¶ -1

deleted file

test/data/ev/test1/test_cases/7/input-7.txt removed ¶ -1

deleted file

test/data/ev/test1/test_cases/8/answer-8.txt removed ¶ -1

deleted file

test/data/ev/test1/test_cases/8/input-8.txt removed ¶ -1

deleted file

test/data/ev/test1/test_cases/8/test.cfg removed ¶ -1

deleted file

test/data/ev/test1/test_cases/9/answer-9.txt removed ¶ -1

deleted file

test/data/ev/test1/test_cases/9/input-9.txt removed ¶ -1

deleted file

test/data/ev/test1/test_cases/all_tests.cfg removed ¶ -39

deleted file

test/data/ev/test2/script/check removed ¶ -59

deleted file

test/data/ev/test2/test_cases/1/answer-1.txt removed ¶ -2

deleted file

test/data/ev/test2/test_cases/1/input-1.txt removed ¶ -2

deleted file

test/data/ev/test2/test_cases/2/answer-2.txt removed ¶ -2

deleted file

test/data/ev/test2/test_cases/2/input-2.txt removed ¶ -2

deleted file

test/data/ev/test2/test_cases/all_tests.cfg removed ¶ -20

deleted file

	Site-wide shortcuts
/	Use quick search box
g h	Goto home page
g g	Goto my private gists page
g G	Goto my public gists page
n r	New repository page
n g	New gist page

	Repositories
g s	Goto summary page
g c	Goto changelog page
g f	Goto files page
g F	Goto files page with file search activated
g p	Goto pull requests page
g o	Goto repository settings
g O	Goto repository permissions settings

Sign in to your account

Author

r23:197d5085d55a - Tue, 11 Mar 2008 04:23:17 - 98 files changed: 1441 inserted, 380 deleted

Sign in to your account

Commit r23:197d5085d55a public

Author

r23:197d5085d55a - Tue, 11 Mar 2008 04:23:17 - 98 files changed: 1441 inserted, 380 deleted

Commit `r23:197d5085d55a` public