Rework the stats script
- Create a new object for each test-case/deployment - Simplify the stats, no need to use defaultdict anymore - Define the start_time to be the one of the test-case/deployment not the start time of the script (influences the requests per minute count) Signed-off-by: Pierre-Yves Chibon <pingou@pingoured.fr>
This commit is contained in:
Pierre-Yves Chibon
parent
bb9779faf3
commit
2149e8a446
1 changed files with 100 additions and 90 deletions
|
|
@ -66,14 +66,14 @@ class TestAPI:
|
|||
|
||||
# request per second
|
||||
# self.rps_min = 0
|
||||
self.rps_mean = collections.defaultdict(int)
|
||||
self.rps_mean = 0
|
||||
# self.rps_max = 0
|
||||
self.tested_requests = collections.defaultdict(int)
|
||||
self.tested_time = collections.defaultdict(int)
|
||||
self.pass_requests = collections.defaultdict(int)
|
||||
self.tested_requests = 0
|
||||
self.tested_time = 0
|
||||
self.pass_requests = 0
|
||||
# failures
|
||||
self.fail_requests = collections.defaultdict(int)
|
||||
self.exception_requests = collections.defaultdict(int)
|
||||
self.fail_requests = 0
|
||||
self.exception_requests = 0
|
||||
|
||||
self.total_tested_requests = 0
|
||||
self.total_tested_time = 0
|
||||
|
|
@ -83,10 +83,10 @@ class TestAPI:
|
|||
self.total_exception_requests = 0
|
||||
|
||||
# time per request
|
||||
self.tpr_min = collections.defaultdict(int)
|
||||
self.tpr_mean = collections.defaultdict(int)
|
||||
self.tpr_max = collections.defaultdict(int)
|
||||
self.sum_response_time = collections.defaultdict(int)
|
||||
self.tpr_min = 0
|
||||
self.tpr_mean = 0
|
||||
self.tpr_max = 0
|
||||
self.sum_response_time = 0
|
||||
|
||||
self.jobs = set()
|
||||
|
||||
|
|
@ -148,7 +148,7 @@ class TestAPI:
|
|||
url = f"{base_url}/id?id=2019-cc9e2d43-6b17-4125-a460-9257b0e52d84"
|
||||
return self._process_reponse(self.get(url))
|
||||
|
||||
def loop_test(self, base_url, loop_wait=0, loop_times=sys.maxsize):
|
||||
def loop_test(self, base_url, name, loop_wait=0, loop_times=sys.maxsize):
|
||||
"""
|
||||
loop test of some APIs for performance test purpose.
|
||||
|
||||
|
|
@ -156,53 +156,55 @@ class TestAPI:
|
|||
loop_wait wait time between two loops.
|
||||
loop_times number of loops, default indefinite
|
||||
"""
|
||||
for name, function in [
|
||||
("test_plain_raw", self.test_plain_raw),
|
||||
("test_filter_by_topic", self.test_filter_by_topic),
|
||||
("test_filter_by_username", self.test_filter_by_username),
|
||||
("test_filter_by_category", self.test_filter_by_category),
|
||||
("test_filter_by_package", self.test_filter_by_package),
|
||||
("test_get_by_id", self.test_get_by_id),
|
||||
]:
|
||||
looped_times = 0
|
||||
functions = {
|
||||
"test_plain_raw": self.test_plain_raw,
|
||||
"test_filter_by_topic": self.test_filter_by_topic,
|
||||
"test_filter_by_username": self.test_filter_by_username,
|
||||
"test_filter_by_category": self.test_filter_by_category,
|
||||
"test_filter_by_package": self.test_filter_by_package,
|
||||
"test_get_by_id": self.test_get_by_id,
|
||||
}
|
||||
looped_times = 0
|
||||
|
||||
while (
|
||||
looped_times < loop_times
|
||||
and not self.event_time_up.is_set()
|
||||
and not self.event_test_done.is_set()
|
||||
):
|
||||
self.jobs.add(name)
|
||||
test_result, elapsed_time = function(base_url)
|
||||
self.queue_results.put([name, test_result, elapsed_time])
|
||||
while (
|
||||
looped_times < loop_times
|
||||
and not self.event_time_up.is_set()
|
||||
and not self.event_test_done.is_set()
|
||||
):
|
||||
self.jobs.add(name)
|
||||
test_result, elapsed_time = functions[name](base_url)
|
||||
self.queue_results.put([name, test_result, elapsed_time])
|
||||
|
||||
looped_times += 1
|
||||
sleep(loop_wait)
|
||||
looped_times += 1
|
||||
sleep(loop_wait)
|
||||
|
||||
def stats(self, env_name=""):
|
||||
""" calculate statistics """
|
||||
end_time = time.time()
|
||||
# get the approximate queue size
|
||||
qsize = self.queue_results.qsize()
|
||||
loop = collections.defaultdict(int)
|
||||
loop = 0
|
||||
for i in range(qsize):
|
||||
try:
|
||||
result = self.queue_results.get_nowait()
|
||||
except Empty:
|
||||
break
|
||||
name = result[0]
|
||||
loop[name] += 1
|
||||
loop += 1
|
||||
# calc stats
|
||||
if result[1] == "exception":
|
||||
self.exception_requests[name] += 1
|
||||
self.exception_requests += 1
|
||||
elif result[1] == "fail":
|
||||
self.fail_requests[name] += 1
|
||||
self.fail_requests += 1
|
||||
if result[2] > self.tpr_max:
|
||||
self.tpr_max = result[2]
|
||||
elif result[1] == "pass":
|
||||
self.pass_requests[name] += 1
|
||||
self.sum_response_time[name] += result[2]
|
||||
if result[2] < self.tpr_min[name]:
|
||||
self.tpr_min[name] = result[2]
|
||||
if result[2] > self.tpr_max[name]:
|
||||
self.tpr_max[name] = result[2]
|
||||
self.pass_requests += 1
|
||||
self.sum_response_time += result[2]
|
||||
if result[2] < self.tpr_min:
|
||||
self.tpr_min = result[2]
|
||||
if result[2] > self.tpr_max:
|
||||
self.tpr_max = result[2]
|
||||
# print stats
|
||||
if env_name:
|
||||
env_name = f" ({env_name})"
|
||||
|
|
@ -212,44 +214,44 @@ class TestAPI:
|
|||
for name in self.jobs:
|
||||
print(f"\n {name}")
|
||||
|
||||
self.tested_requests[name] += loop[name]
|
||||
self.total_tested_requests += self.tested_requests[name]
|
||||
self.total_pass_requests += self.pass_requests[name]
|
||||
self.total_fail_requests += self.fail_requests[name]
|
||||
self.total_exception_requests += self.exception_requests[name]
|
||||
self.tested_requests += loop
|
||||
self.total_tested_requests += self.tested_requests
|
||||
self.total_pass_requests += self.pass_requests
|
||||
self.total_fail_requests += self.fail_requests
|
||||
self.total_exception_requests += self.exception_requests
|
||||
|
||||
# time per requests mean (avg)
|
||||
if self.pass_requests[name] != 0 or self.fail_requests[name] != 0:
|
||||
divided_by = self.pass_requests[name]
|
||||
if self.pass_requests[name] == 0:
|
||||
divided_by = self.fail_requests[name]
|
||||
self.tpr_mean[name] = self.sum_response_time[name] / divided_by
|
||||
if self.pass_requests != 0 or self.fail_requests != 0:
|
||||
divided_by = self.pass_requests
|
||||
if self.pass_requests == 0:
|
||||
divided_by = self.fail_requests
|
||||
self.tpr_mean = self.sum_response_time / divided_by
|
||||
# requests per second
|
||||
if self.start_time == 0:
|
||||
_log.error("stats: self.start_time is not set, skipping rps stats.")
|
||||
else:
|
||||
# calc the tested time so far.
|
||||
tested_time = end_time - self.start_time
|
||||
req = self.pass_requests[name]
|
||||
if self.pass_requests[name] == 0:
|
||||
req = self.fail_requests[name]
|
||||
self.rps_mean[name] = req / tested_time
|
||||
req = self.pass_requests
|
||||
if self.pass_requests == 0:
|
||||
req = self.fail_requests
|
||||
self.rps_mean = req / tested_time
|
||||
|
||||
print(
|
||||
"Requests: %s, pass: %s, fail: %s, exception: %s"
|
||||
% (
|
||||
self.tested_requests[name],
|
||||
self.pass_requests[name],
|
||||
self.fail_requests[name],
|
||||
self.exception_requests[name],
|
||||
self.tested_requests,
|
||||
self.pass_requests,
|
||||
self.fail_requests,
|
||||
self.exception_requests,
|
||||
)
|
||||
)
|
||||
if self.rps_mean[name]:
|
||||
if self.rps_mean:
|
||||
print("For pass requests:")
|
||||
print("Request per Second - mean: %.2f" % self.rps_mean[name])
|
||||
print("Request per Second - mean: %.2f" % self.rps_mean)
|
||||
print(
|
||||
"Time per Request - mean: %.6f, min: %.6f, max: %.6f"
|
||||
% (self.tpr_mean[name], self.tpr_min[name], self.tpr_max[name])
|
||||
% (self.tpr_mean, self.tpr_min, self.tpr_max)
|
||||
)
|
||||
|
||||
print(
|
||||
|
|
@ -341,45 +343,53 @@ def main():
|
|||
for env_name, base_url in [
|
||||
("datagrepper-timescalebd/aws", "http://datagrepper-timescale.arc.fedorainfracloud.org/datagrepper"),
|
||||
("datagrepper-test/aws", "http://datagrepper-test.arc.fedorainfracloud.org/datagrepper"),
|
||||
("datagrepper-timescaledb-array/aws", "http://datagrepper.arc.fedorainfracloud.org/datagrepper"),
|
||||
("datagrepper-prod/aws", "http://datagrepper-adam.arc.fedorainfracloud.org/datagrepper"),
|
||||
("datagrepper-prod/openshift", "https://datagrepper-monitor-dashboard.app.os.fedoraproject.org"),
|
||||
]:
|
||||
perf_test = TestAPI()
|
||||
perf_test.start_time = start_time
|
||||
workers = []
|
||||
for name in [
|
||||
"test_filter_by_topic",
|
||||
"test_plain_raw",
|
||||
"test_filter_by_category",
|
||||
"test_filter_by_username",
|
||||
"test_filter_by_package",
|
||||
"test_get_by_id",
|
||||
]:
|
||||
perf_test = TestAPI()
|
||||
perf_test.start_time = time.time()
|
||||
workers = []
|
||||
|
||||
for i in range(concurrent_users):
|
||||
thread = Thread(
|
||||
target=perf_test.loop_test,
|
||||
kwargs={
|
||||
"base_url": base_url,
|
||||
"loop_times": loop_times,
|
||||
},
|
||||
daemon=True,
|
||||
)
|
||||
thread.start()
|
||||
workers.append(thread)
|
||||
# ramp up wait
|
||||
sleep(ramp_up / concurrent_users)
|
||||
for i in range(concurrent_users):
|
||||
thread = Thread(
|
||||
target=perf_test.loop_test,
|
||||
kwargs={
|
||||
"base_url": base_url,
|
||||
"loop_times": loop_times,
|
||||
"name": name,
|
||||
},
|
||||
daemon=True,
|
||||
)
|
||||
thread.start()
|
||||
workers.append(thread)
|
||||
# ramp up wait
|
||||
sleep(ramp_up / concurrent_users)
|
||||
|
||||
# start timer
|
||||
perf_test.start_timer(test_time)
|
||||
# start timer
|
||||
perf_test.start_timer(test_time)
|
||||
|
||||
# Block until all threads finish.
|
||||
for w in workers:
|
||||
w.join()
|
||||
# Block until all threads finish.
|
||||
for w in workers:
|
||||
w.join()
|
||||
|
||||
# clean up
|
||||
# stop timer if loop_times is reached first.
|
||||
perf_test.cancel_timer()
|
||||
# clean up
|
||||
# stop timer if loop_times is reached first.
|
||||
perf_test.cancel_timer()
|
||||
|
||||
perf_test.end_time = time.time()
|
||||
perf_test.end_time = time.time()
|
||||
|
||||
# Ensure to execute the last statistics:
|
||||
perf_test.stats(env_name=env_name)
|
||||
# Ensure to execute the last statistics:
|
||||
perf_test.stats(env_name=env_name)
|
||||
|
||||
perf_test.reset_stats()
|
||||
del(perf_test)
|
||||
|
||||
print(
|
||||
"\nTests ended at %s.\nTotal test time: %s seconds."
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue