From 2149e8a446e838507e0b7911e90c191031211021 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Tue, 16 Feb 2021 23:18:58 +0100 Subject: [PATCH] Rework the stats script - Create a new object for each test-case/deployment - Simplify the stats, no need to use defaultdict anymore - Define the start_time to be the one of the test-case/deployment not the start time of the script (influences the requests per minute count) Signed-off-by: Pierre-Yves Chibon --- scripts/test_datagrepper_perfs.py | 190 ++++++++++++++++-------------- 1 file changed, 100 insertions(+), 90 deletions(-) diff --git a/scripts/test_datagrepper_perfs.py b/scripts/test_datagrepper_perfs.py index 7825893..1645312 100644 --- a/scripts/test_datagrepper_perfs.py +++ b/scripts/test_datagrepper_perfs.py @@ -66,14 +66,14 @@ class TestAPI: # request per second # self.rps_min = 0 - self.rps_mean = collections.defaultdict(int) + self.rps_mean = 0 # self.rps_max = 0 - self.tested_requests = collections.defaultdict(int) - self.tested_time = collections.defaultdict(int) - self.pass_requests = collections.defaultdict(int) + self.tested_requests = 0 + self.tested_time = 0 + self.pass_requests = 0 # failures - self.fail_requests = collections.defaultdict(int) - self.exception_requests = collections.defaultdict(int) + self.fail_requests = 0 + self.exception_requests = 0 self.total_tested_requests = 0 self.total_tested_time = 0 @@ -83,10 +83,10 @@ class TestAPI: self.total_exception_requests = 0 # time per request - self.tpr_min = collections.defaultdict(int) - self.tpr_mean = collections.defaultdict(int) - self.tpr_max = collections.defaultdict(int) - self.sum_response_time = collections.defaultdict(int) + self.tpr_min = 0 + self.tpr_mean = 0 + self.tpr_max = 0 + self.sum_response_time = 0 self.jobs = set() @@ -148,7 +148,7 @@ class TestAPI: url = f"{base_url}/id?id=2019-cc9e2d43-6b17-4125-a460-9257b0e52d84" return self._process_reponse(self.get(url)) - def loop_test(self, base_url, loop_wait=0, loop_times=sys.maxsize): + def loop_test(self, base_url, name, loop_wait=0, loop_times=sys.maxsize): """ loop test of some APIs for performance test purpose. @@ -156,53 +156,55 @@ class TestAPI: loop_wait wait time between two loops. loop_times number of loops, default indefinite """ - for name, function in [ - ("test_plain_raw", self.test_plain_raw), - ("test_filter_by_topic", self.test_filter_by_topic), - ("test_filter_by_username", self.test_filter_by_username), - ("test_filter_by_category", self.test_filter_by_category), - ("test_filter_by_package", self.test_filter_by_package), - ("test_get_by_id", self.test_get_by_id), - ]: - looped_times = 0 + functions = { + "test_plain_raw": self.test_plain_raw, + "test_filter_by_topic": self.test_filter_by_topic, + "test_filter_by_username": self.test_filter_by_username, + "test_filter_by_category": self.test_filter_by_category, + "test_filter_by_package": self.test_filter_by_package, + "test_get_by_id": self.test_get_by_id, + } + looped_times = 0 - while ( - looped_times < loop_times - and not self.event_time_up.is_set() - and not self.event_test_done.is_set() - ): - self.jobs.add(name) - test_result, elapsed_time = function(base_url) - self.queue_results.put([name, test_result, elapsed_time]) + while ( + looped_times < loop_times + and not self.event_time_up.is_set() + and not self.event_test_done.is_set() + ): + self.jobs.add(name) + test_result, elapsed_time = functions[name](base_url) + self.queue_results.put([name, test_result, elapsed_time]) - looped_times += 1 - sleep(loop_wait) + looped_times += 1 + sleep(loop_wait) def stats(self, env_name=""): """ calculate statistics """ end_time = time.time() # get the approximate queue size qsize = self.queue_results.qsize() - loop = collections.defaultdict(int) + loop = 0 for i in range(qsize): try: result = self.queue_results.get_nowait() except Empty: break name = result[0] - loop[name] += 1 + loop += 1 # calc stats if result[1] == "exception": - self.exception_requests[name] += 1 + self.exception_requests += 1 elif result[1] == "fail": - self.fail_requests[name] += 1 + self.fail_requests += 1 + if result[2] > self.tpr_max: + self.tpr_max = result[2] elif result[1] == "pass": - self.pass_requests[name] += 1 - self.sum_response_time[name] += result[2] - if result[2] < self.tpr_min[name]: - self.tpr_min[name] = result[2] - if result[2] > self.tpr_max[name]: - self.tpr_max[name] = result[2] + self.pass_requests += 1 + self.sum_response_time += result[2] + if result[2] < self.tpr_min: + self.tpr_min = result[2] + if result[2] > self.tpr_max: + self.tpr_max = result[2] # print stats if env_name: env_name = f" ({env_name})" @@ -212,44 +214,44 @@ class TestAPI: for name in self.jobs: print(f"\n {name}") - self.tested_requests[name] += loop[name] - self.total_tested_requests += self.tested_requests[name] - self.total_pass_requests += self.pass_requests[name] - self.total_fail_requests += self.fail_requests[name] - self.total_exception_requests += self.exception_requests[name] + self.tested_requests += loop + self.total_tested_requests += self.tested_requests + self.total_pass_requests += self.pass_requests + self.total_fail_requests += self.fail_requests + self.total_exception_requests += self.exception_requests # time per requests mean (avg) - if self.pass_requests[name] != 0 or self.fail_requests[name] != 0: - divided_by = self.pass_requests[name] - if self.pass_requests[name] == 0: - divided_by = self.fail_requests[name] - self.tpr_mean[name] = self.sum_response_time[name] / divided_by + if self.pass_requests != 0 or self.fail_requests != 0: + divided_by = self.pass_requests + if self.pass_requests == 0: + divided_by = self.fail_requests + self.tpr_mean = self.sum_response_time / divided_by # requests per second if self.start_time == 0: _log.error("stats: self.start_time is not set, skipping rps stats.") else: # calc the tested time so far. tested_time = end_time - self.start_time - req = self.pass_requests[name] - if self.pass_requests[name] == 0: - req = self.fail_requests[name] - self.rps_mean[name] = req / tested_time + req = self.pass_requests + if self.pass_requests == 0: + req = self.fail_requests + self.rps_mean = req / tested_time print( "Requests: %s, pass: %s, fail: %s, exception: %s" % ( - self.tested_requests[name], - self.pass_requests[name], - self.fail_requests[name], - self.exception_requests[name], + self.tested_requests, + self.pass_requests, + self.fail_requests, + self.exception_requests, ) ) - if self.rps_mean[name]: + if self.rps_mean: print("For pass requests:") - print("Request per Second - mean: %.2f" % self.rps_mean[name]) + print("Request per Second - mean: %.2f" % self.rps_mean) print( "Time per Request - mean: %.6f, min: %.6f, max: %.6f" - % (self.tpr_mean[name], self.tpr_min[name], self.tpr_max[name]) + % (self.tpr_mean, self.tpr_min, self.tpr_max) ) print( @@ -341,45 +343,53 @@ def main(): for env_name, base_url in [ ("datagrepper-timescalebd/aws", "http://datagrepper-timescale.arc.fedorainfracloud.org/datagrepper"), ("datagrepper-test/aws", "http://datagrepper-test.arc.fedorainfracloud.org/datagrepper"), - ("datagrepper-timescaledb-array/aws", "http://datagrepper.arc.fedorainfracloud.org/datagrepper"), ("datagrepper-prod/aws", "http://datagrepper-adam.arc.fedorainfracloud.org/datagrepper"), ("datagrepper-prod/openshift", "https://datagrepper-monitor-dashboard.app.os.fedoraproject.org"), ]: - perf_test = TestAPI() - perf_test.start_time = start_time - workers = [] + for name in [ + "test_filter_by_topic", + "test_plain_raw", + "test_filter_by_category", + "test_filter_by_username", + "test_filter_by_package", + "test_get_by_id", + ]: + perf_test = TestAPI() + perf_test.start_time = time.time() + workers = [] - for i in range(concurrent_users): - thread = Thread( - target=perf_test.loop_test, - kwargs={ - "base_url": base_url, - "loop_times": loop_times, - }, - daemon=True, - ) - thread.start() - workers.append(thread) - # ramp up wait - sleep(ramp_up / concurrent_users) + for i in range(concurrent_users): + thread = Thread( + target=perf_test.loop_test, + kwargs={ + "base_url": base_url, + "loop_times": loop_times, + "name": name, + }, + daemon=True, + ) + thread.start() + workers.append(thread) + # ramp up wait + sleep(ramp_up / concurrent_users) - # start timer - perf_test.start_timer(test_time) + # start timer + perf_test.start_timer(test_time) - # Block until all threads finish. - for w in workers: - w.join() + # Block until all threads finish. + for w in workers: + w.join() - # clean up - # stop timer if loop_times is reached first. - perf_test.cancel_timer() + # clean up + # stop timer if loop_times is reached first. + perf_test.cancel_timer() - perf_test.end_time = time.time() + perf_test.end_time = time.time() - # Ensure to execute the last statistics: - perf_test.stats(env_name=env_name) + # Ensure to execute the last statistics: + perf_test.stats(env_name=env_name) - perf_test.reset_stats() + del(perf_test) print( "\nTests ended at %s.\nTotal test time: %s seconds."