Infrastructure/arc
1
0
Fork 0
Code Issues 1 Pull requests Projects Releases Packages Wiki Activity Actions

Rework the stats script

Browse source 
- Create a new object for each test-case/deployment
- Simplify the stats, no need to use defaultdict anymore
- Define the start_time to be the one of the test-case/deployment not
  the start time of the script (influences the requests per minute
  count)

Signed-off-by: Pierre-Yves Chibon <pingou@pingoured.fr>
This commit is contained in:
Pierre-Yves Chibon 2021-02-16 23:18:58 +01:00
parent bb9779faf3
commit 2149e8a446
1 changed files with 100 additions and 90 deletions
Download patch file Download diff file Expand all files Collapse all files

190
scripts/test_datagrepper_perfs.py
View file

@ -66,14 +66,14 @@ class TestAPI:
# request per second # request per second
# self.rps_min = 0 # self.rps_min = 0
self.rps_mean = collections.defaultdict(int) self.rps_mean = 0
# self.rps_max = 0 # self.rps_max = 0
self.tested_requests = collections.defaultdict(int) self.tested_requests = 0
self.tested_time = collections.defaultdict(int) self.tested_time = 0
self.pass_requests = collections.defaultdict(int) self.pass_requests = 0
# failures # failures
self.fail_requests = collections.defaultdict(int) self.fail_requests = 0
self.exception_requests = collections.defaultdict(int) self.exception_requests = 0
self.total_tested_requests = 0 self.total_tested_requests = 0
self.total_tested_time = 0 self.total_tested_time = 0
@ -83,10 +83,10 @@ class TestAPI:
self.total_exception_requests = 0 self.total_exception_requests = 0
# time per request # time per request
self.tpr_min = collections.defaultdict(int) self.tpr_min = 0
self.tpr_mean = collections.defaultdict(int) self.tpr_mean = 0
self.tpr_max = collections.defaultdict(int) self.tpr_max = 0
self.sum_response_time = collections.defaultdict(int) self.sum_response_time = 0
self.jobs = set() self.jobs = set()
@ -148,7 +148,7 @@ class TestAPI:
url = f"{base_url}/id?id=2019-cc9e2d43-6b17-4125-a460-9257b0e52d84" url = f"{base_url}/id?id=2019-cc9e2d43-6b17-4125-a460-9257b0e52d84"
return self._process_reponse(self.get(url)) return self._process_reponse(self.get(url))
def loop_test(self, base_url, loop_wait=0, loop_times=sys.maxsize): def loop_test(self, base_url, name, loop_wait=0, loop_times=sys.maxsize):
""" """
loop test of some APIs for performance test purpose. loop test of some APIs for performance test purpose.
@ -156,53 +156,55 @@ class TestAPI:
loop_wait wait time between two loops. loop_wait wait time between two loops.
loop_times number of loops, default indefinite loop_times number of loops, default indefinite
""" """
for name, function in [ functions = {
("test_plain_raw", self.test_plain_raw), "test_plain_raw": self.test_plain_raw,
("test_filter_by_topic", self.test_filter_by_topic), "test_filter_by_topic": self.test_filter_by_topic,
("test_filter_by_username", self.test_filter_by_username), "test_filter_by_username": self.test_filter_by_username,
("test_filter_by_category", self.test_filter_by_category), "test_filter_by_category": self.test_filter_by_category,
("test_filter_by_package", self.test_filter_by_package), "test_filter_by_package": self.test_filter_by_package,
("test_get_by_id", self.test_get_by_id), "test_get_by_id": self.test_get_by_id,
]: }
looped_times = 0 looped_times = 0
while ( while (
looped_times < loop_times looped_times < loop_times
and not self.event_time_up.is_set() and not self.event_time_up.is_set()
and not self.event_test_done.is_set() and not self.event_test_done.is_set()
): ):
self.jobs.add(name) self.jobs.add(name)
test_result, elapsed_time = function(base_url) test_result, elapsed_time = functions[name](base_url)
self.queue_results.put([name, test_result, elapsed_time]) self.queue_results.put([name, test_result, elapsed_time])
looped_times += 1 looped_times += 1
sleep(loop_wait) sleep(loop_wait)
def stats(self, env_name=""): def stats(self, env_name=""):
""" calculate statistics """ """ calculate statistics """
end_time = time.time() end_time = time.time()
# get the approximate queue size # get the approximate queue size
qsize = self.queue_results.qsize() qsize = self.queue_results.qsize()
loop = collections.defaultdict(int) loop = 0
for i in range(qsize): for i in range(qsize):
try: try:
result = self.queue_results.get_nowait() result = self.queue_results.get_nowait()
except Empty: except Empty:
break break
name = result[0] name = result[0]
loop[name] += 1 loop += 1
# calc stats # calc stats
if result[1] == "exception": if result[1] == "exception":
self.exception_requests[name] += 1 self.exception_requests += 1
elif result[1] == "fail": elif result[1] == "fail":
self.fail_requests[name] += 1 self.fail_requests += 1
if result[2] > self.tpr_max:
self.tpr_max = result[2]
elif result[1] == "pass": elif result[1] == "pass":
self.pass_requests[name] += 1 self.pass_requests += 1
self.sum_response_time[name] += result[2] self.sum_response_time += result[2]
if result[2] < self.tpr_min[name]: if result[2] < self.tpr_min:
self.tpr_min[name] = result[2] self.tpr_min = result[2]
if result[2] > self.tpr_max[name]: if result[2] > self.tpr_max:
self.tpr_max[name] = result[2] self.tpr_max = result[2]
# print stats # print stats
if env_name: if env_name:
env_name = f" ({env_name})" env_name = f" ({env_name})"
@ -212,44 +214,44 @@ class TestAPI:
for name in self.jobs: for name in self.jobs:
print(f"\n {name}") print(f"\n {name}")
self.tested_requests[name] += loop[name] self.tested_requests += loop
self.total_tested_requests += self.tested_requests[name] self.total_tested_requests += self.tested_requests
self.total_pass_requests += self.pass_requests[name] self.total_pass_requests += self.pass_requests
self.total_fail_requests += self.fail_requests[name] self.total_fail_requests += self.fail_requests
self.total_exception_requests += self.exception_requests[name] self.total_exception_requests += self.exception_requests
# time per requests mean (avg) # time per requests mean (avg)
if self.pass_requests[name] != 0 or self.fail_requests[name] != 0: if self.pass_requests != 0 or self.fail_requests != 0:
divided_by = self.pass_requests[name] divided_by = self.pass_requests
if self.pass_requests[name] == 0: if self.pass_requests == 0:
divided_by = self.fail_requests[name] divided_by = self.fail_requests
self.tpr_mean[name] = self.sum_response_time[name] / divided_by self.tpr_mean = self.sum_response_time / divided_by
# requests per second # requests per second
if self.start_time == 0: if self.start_time == 0:
_log.error("stats: self.start_time is not set, skipping rps stats.") _log.error("stats: self.start_time is not set, skipping rps stats.")
else: else:
# calc the tested time so far. # calc the tested time so far.
tested_time = end_time - self.start_time tested_time = end_time - self.start_time
req = self.pass_requests[name] req = self.pass_requests
if self.pass_requests[name] == 0: if self.pass_requests == 0:
req = self.fail_requests[name] req = self.fail_requests
self.rps_mean[name] = req / tested_time self.rps_mean = req / tested_time
print( print(
"Requests: %s, pass: %s, fail: %s, exception: %s" "Requests: %s, pass: %s, fail: %s, exception: %s"
% ( % (
self.tested_requests[name], self.tested_requests,
self.pass_requests[name], self.pass_requests,
self.fail_requests[name], self.fail_requests,
self.exception_requests[name], self.exception_requests,
) )
) )
if self.rps_mean[name]: if self.rps_mean:
print("For pass requests:") print("For pass requests:")
print("Request per Second - mean: %.2f" % self.rps_mean[name]) print("Request per Second - mean: %.2f" % self.rps_mean)
print( print(
"Time per Request - mean: %.6f, min: %.6f, max: %.6f" "Time per Request - mean: %.6f, min: %.6f, max: %.6f"
% (self.tpr_mean[name], self.tpr_min[name], self.tpr_max[name]) % (self.tpr_mean, self.tpr_min, self.tpr_max)
) )
print( print(
@ -341,45 +343,53 @@ def main():
for env_name, base_url in [ for env_name, base_url in [
("datagrepper-timescalebd/aws", "http://datagrepper-timescale.arc.fedorainfracloud.org/datagrepper"), ("datagrepper-timescalebd/aws", "http://datagrepper-timescale.arc.fedorainfracloud.org/datagrepper"),
("datagrepper-test/aws", "http://datagrepper-test.arc.fedorainfracloud.org/datagrepper"), ("datagrepper-test/aws", "http://datagrepper-test.arc.fedorainfracloud.org/datagrepper"),
("datagrepper-timescaledb-array/aws", "http://datagrepper.arc.fedorainfracloud.org/datagrepper"),
("datagrepper-prod/aws", "http://datagrepper-adam.arc.fedorainfracloud.org/datagrepper"), ("datagrepper-prod/aws", "http://datagrepper-adam.arc.fedorainfracloud.org/datagrepper"),
("datagrepper-prod/openshift", "https://datagrepper-monitor-dashboard.app.os.fedoraproject.org"), ("datagrepper-prod/openshift", "https://datagrepper-monitor-dashboard.app.os.fedoraproject.org"),
]: ]:
perf_test = TestAPI() for name in [
perf_test.start_time = start_time "test_filter_by_topic",
workers = [] "test_plain_raw",
"test_filter_by_category",
"test_filter_by_username",
"test_filter_by_package",
"test_get_by_id",
]:
perf_test = TestAPI()
perf_test.start_time = time.time()
workers = []
for i in range(concurrent_users): for i in range(concurrent_users):
thread = Thread( thread = Thread(
target=perf_test.loop_test, target=perf_test.loop_test,
kwargs={ kwargs={
"base_url": base_url, "base_url": base_url,
"loop_times": loop_times, "loop_times": loop_times,
}, "name": name,
daemon=True, },
) daemon=True,
thread.start() )
workers.append(thread) thread.start()
# ramp up wait workers.append(thread)
sleep(ramp_up / concurrent_users) # ramp up wait
sleep(ramp_up / concurrent_users)
# start timer # start timer
perf_test.start_timer(test_time) perf_test.start_timer(test_time)
# Block until all threads finish. # Block until all threads finish.
for w in workers: for w in workers:
w.join() w.join()
# clean up # clean up
# stop timer if loop_times is reached first. # stop timer if loop_times is reached first.
perf_test.cancel_timer() perf_test.cancel_timer()
perf_test.end_time = time.time() perf_test.end_time = time.time()
# Ensure to execute the last statistics: # Ensure to execute the last statistics:
perf_test.stats(env_name=env_name) perf_test.stats(env_name=env_name)
perf_test.reset_stats() del(perf_test)
print( print(
"\nTests ended at %s.\nTotal test time: %s seconds." "\nTests ended at %s.\nTotal test time: %s seconds."