master
1import difflib
2import os.path
3from random import SystemRandom
4import re
5import subprocess
6import sys
7import time
8
9# From this package
10import lib.error as error
11import lib.http_client as http_client
12import lib.util as util
13
14data_directory = os.path.join(os.path.dirname(__file__), "..", "data")
15
16class TestCase(object):
17 def __init__(self, harness, id_or_url):
18 self.harness = harness
19 self.id, self.url = self.normalize_id_and_url(id_or_url)
20 self.json = None
21
22 def normalize_id_and_url(self, id_or_url):
23 if re.match("\Ahttps?:", id_or_url):
24 url = id_or_url
25 # Look at the last component and remove extension
26 id = id_or_url.split('/')[-1].split('.')[0]
27 else:
28 id = id_or_url
29 level = self.harness.LEVEL
30 url = "https://stripe-ctf-3.s3.amazonaws.com/level%s/%s.json" % (level, id)
31 return id, url
32
33 def dump_path(self):
34 return os.path.join(self.harness.test_cases_path(), self.id + ".json")
35
36 def load(self):
37 if self.json: return self.json
38 try:
39 f = open(self.dump_path(), "r")
40 self.json = util.json.load(f)
41 f.close()
42 return self.json
43 except IOError:
44 pass
45 util.logger.info('Fetching. URL: %s', self.url)
46 content = self.harness.fetch_s3_resource(self.url)
47 try:
48 self.json = util.json.loads(content)
49 except ValueError:
50 # Decoding the JSON failed.
51 msg = ("There was a problem parsing the test case. We expected "
52 "JSON. We received: %s" % (content,))
53 raise error.StripeError(msg)
54 return self.json
55
56 def flush(self):
57 f = open(os.path.join(self.harness.test_cases_path(), self.id + ".json"), "w")
58 util.json.dump(self.json, f)
59 f.close()
60
61class AbstractHarness(object):
62 def __init__(self, ids_or_urls=[], options={}):
63 util.mkdir_p(self.test_cases_path())
64 if not os.path.isfile(http_client.certs_path()):
65 msg = ("You seem to have deleted the file of certificates "
66 "that shipped with this repo. It should exist "
67 "at %s" % http_client.certs_path())
68 raise error.StripeError(msg)
69 if ids_or_urls == []:
70 util.logger.info('No test case supplied. Randomly choosing among defaults.')
71 ids_or_urls = [SystemRandom().choice(self.DEFAULT_TEST_CASES)]
72 self.test_cases = map(lambda token: TestCase(self, token), ids_or_urls)
73 self.options = options
74 headers = {
75 'User-Agent': 'Stripe TestHarness/%s' % (self.VERSION,),
76 }
77 self.http_client = http_client.new_default_http_client(headers=headers, verify_ssl_certs=True)
78
79 def fetch_s3_resource(self, url):
80 try:
81 content, status_code = self.http_client.request("get", url)
82 except error.HTTPConnectionError:
83 err = util.exception_as()
84 msg = ("There was an error while connecting to fetch "
85 "the url %s. Please check your connectivity. If there "
86 "continues to be an issue, please let us know at "
87 "ctf@stripe.com. The specific error is:\n" % (url,))
88 raise error.StripeError(msg + str(err))
89 if status_code == 200:
90 return content
91 elif status_code == 403:
92 msg = ("We received a 403 while fetching the url %s. "
93 "This probably means that you are trying to get "
94 "something that doesn't actually exist." % (url,))
95 raise error.StripeError(msg)
96 else:
97 msg = ("We received the unexpected response code %i while "
98 "fetching the url %s." % (status_code, url,))
99 raise error.StripeError(msg)
100
101 def run(self):
102 task = self.options["task"]
103
104 if task == "execute":
105 test_cases_to_execute = self.load_test_cases()
106 self.execute(test_cases_to_execute)
107 else:
108 raise StandardError("Unrecognized task " + task)
109
110 def test_cases_path(self):
111 return os.path.join(
112 data_directory,
113 "downloaded_test_cases",
114 "version%i" % self.VERSION)
115
116 def flush_test_cases(self):
117 util.logger.info('Flushing. Path: %s', self.test_cases_path())
118 for test_case in self.test_cases:
119 test_case.flush(self.test_cases_path())
120
121 def add_test_case(self, test_case):
122 self.test_cases.append(test_case)
123
124 def load_test_cases(self):
125 loaded_test_cases = []
126 for test_case in self.test_cases:
127 result = test_case.load()
128 if not result: continue
129 test_case.flush()
130 loaded_test_cases.append(test_case)
131 return loaded_test_cases
132
133 def hook_preexecute(self):
134 # May override
135 pass
136
137 def execute(self, test_cases_to_execute):
138 self.hook_preexecute()
139 runner = self.hook_create_runner()
140
141 for test_case in test_cases_to_execute:
142 if self.options["raw"]:
143 util.logger.info(runner.run_test_case_raw(test_case.json))
144 else:
145 runner.run_test_case(test_case.json)
146
147class AbstractRunner(object):
148 def __init__(self, options):
149 pass
150
151 # may override
152 def code_directory(self):
153 return os.path.join(os.path.dirname(__file__), "..")
154
155 def log_diff(self, benchmark_output, user_output):
156 util.logger.info("Here is the head of your output:")
157 util.logger.info(user_output[0:1000])
158 diff = list(difflib.Differ().compare(benchmark_output.splitlines(True),
159 user_output.splitlines(True)))
160 lines = filter(lambda line: line[0] != "?", diff[0:20])
161 util.logger.info("\n***********\n")
162 util.logger.info("Here is the head of the diff between your output and the benchmark:")
163 util.logger.info("".join(lines))
164
165 def run_build_sh(self):
166 util.logger.info("Building your code via `build.sh`.")
167 build_runner = subprocess.Popen([
168 os.path.join(self.code_directory(), "build.sh")],
169 stdout=subprocess.PIPE, stderr=subprocess.PIPE)
170 # Blocks
171 stdout, stderr = build_runner.communicate()
172 if build_runner.returncode == 0:
173 util.logger.info("Done building your code.")
174 else:
175 util.logger.info("Build failed with code %i. Stderr:", build_runner.returncode)
176 util.logger.info(stderr)
177
178 # may override
179 def hook_prerun(self):
180 pass
181
182 def run_test_case(self, test_case):
183 self.hook_prerun()
184 id = test_case['id']
185 util.logger.info("About to run test case: %s" % id)
186 input = test_case['input']
187 result = self.run_input(input)
188 return self.report_result(test_case, result)
189
190 def run_test_case_raw(self, test_case):
191 self.hook_prerun()
192 input = test_case['input']
193 result = self.run_input(input)
194 return result['output']
195
196 def run_input(self, input):
197 util.logger.info("Beginning run.")
198 output = self.run_subprocess_command(self.subprocess_command(), input)
199 util.logger.info('Finished run')
200 return output
201
202 def report_stderr(self, stderr):
203 if not stderr: return
204 util.logger.info("Standard error from trial run:")
205 util.logger.info(stderr)
206
207 def subprocess_communicate(self, runner, input):
208 if sys.version_info >= (3, 0):
209 input = input.encode('utf-8')
210 stdout, stderr = runner.communicate(input)
211 if sys.version_info >= (3, 0):
212 stderr = stderr.decode('utf-8')
213 stdout = stdout.decode('utf-8')
214 return stdout, stderr
215
216 def run_subprocess_command(self, command, input):
217 start_time = time.time()
218 runner = subprocess.Popen(command, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
219 stdout, stderr = self.subprocess_communicate(runner, input)
220 end_time = time.time()
221 return {
222 'wall_clock_time': end_time - start_time,
223 'output': stdout,
224 'input': input,
225 'level': self.LEVEL,
226 'exitstatus': runner.returncode,
227 }