master
1import os.path
2import subprocess
3
4# From this package
5import lib.test_framework as test_framework
6import lib.util as util
7
8class Runner(test_framework.AbstractRunner):
9 LEVEL = 2
10
11 def __init__(self, options):
12 super(Runner, self).__init__(options)
13 self.secret = util.random_letters(16)
14 self.client_port = "3000"
15 self.backend_ports = ["3001", "3002"]
16 self.results_path = os.path.join(
17 test_framework.data_directory,
18 "results-%s.json" % self.secret)
19
20 def code_directory(self):
21 return os.path.join(os.path.dirname(__file__), "..")
22
23 def hook_prerun(self):
24 self.run_build_sh()
25
26 def read_result_file(self, path):
27 try:
28 f = open(path)
29 except IOError:
30 return None
31 results = util.json.load(f)
32 f.close()
33 return results
34
35 def score(self, results):
36 return max(0.01, results['good_responses'] - results['backend_deficit'] / 8.0)
37
38 def spinup_backend(self, port):
39 return subprocess.Popen([
40 os.path.join(self.code_directory(), "network_simulation", "backend.js"),
41 "--secret", self.secret,
42 "--in-port", port])
43
44 # overrides
45 def run_input(self, input):
46 util.logger.info("Beginning run.")
47 backend_runners = []
48 for port in self.backend_ports:
49 backend_runners.append(self.spinup_backend(port))
50 shield_runner = subprocess.Popen([
51 os.path.join(self.code_directory(), "shield"),
52 "--in-port", self.client_port,
53 "--out-ports", ",".join(self.backend_ports)])
54 sword_runner = subprocess.Popen([
55 os.path.join(self.code_directory(), "network_simulation", "sword.js"),
56 "--secret", self.secret,
57 "--out-port", self.client_port,
58 "--results-path", self.results_path, input],
59 stdin=subprocess.PIPE)
60 # Blocks:
61 stdout, stderr = sword_runner.communicate()
62 for br in backend_runners: br.terminate()
63 shield_runner.terminate()
64 util.logger.info('Finished run')
65 results = self.read_result_file(self.results_path)
66 if results != None:
67 output_dictionary = {
68 'score': self.score(results),
69 'good_responses': results['good_responses'],
70 'backend_deficit': results['backend_deficit'],
71 'correct': results['correct'],
72 'results': results
73 }
74 else:
75 output_dictionary = {
76 'correct': False,
77 'unclean_description': "`sword.js` did not write a results file"
78 }
79 output_dictionary.update({
80 'input': input,
81 'level': self.LEVEL,
82 'exitstatus': sword_runner.returncode,
83 })
84 return output_dictionary
85
86 def report_result(self, test_case, result):
87 returncode = result['exitstatus']
88
89 if returncode != 0:
90 util.logger.info('Your `shield` exited uncleanly. Exit code: %i',
91 returncode)
92 elif not result['correct']:
93 util.logger.error("Test case failed. Reason: %s", result['unclean_description'])
94 else:
95 benchmark_score = test_case['score']
96 your_score = result['score']
97 score_ratio = (your_score + 0.0) / benchmark_score
98 your_good_responses = result['results']['good_responses']
99 your_deficit = round(result['results']['backend_deficit'] / 8.0, 2)
100 benchmark_good_responses = test_case['results']['good_responses']
101 benchmark_deficit = round(test_case['results']['backend_deficit'] / 8.0, 2)
102 msg = ("Test case passed. Your score: %(your_score)f. Benchmark score: "
103 "%(benchmark_score)f. You/Benchmark: %(score_ratio)f."
104 "You handled %(your_good_responses)s legitimate responses and "
105 "you received %(your_deficit)s negative points for idle time "
106 "on the backends. The benchmark handled %(benchmark_good_responses)s "
107 "and received %(benchmark_deficit)s negative points.")
108 util.logger.info(msg,
109 {"your_score": your_score,
110 "benchmark_score": benchmark_score,
111 "score_ratio": score_ratio,
112 "your_good_responses": your_good_responses,
113 "your_deficit": your_deficit,
114 "benchmark_good_responses": benchmark_good_responses,
115 "benchmark_deficit": benchmark_deficit})