infrastructure/ansible
2
0
Fork 0
Code Pull requests 7 Activity Actions

logview: migrate optparse to argparse, add additional date filtering capabilities, and some pep8 fixes

Browse source
This commit is contained in:
Francois Andrieu 2020-10-28 16:45:29 +01:00
parent 88b07454b7
commit 5e91005e93
2 changed files with 110 additions and 71 deletions
Download patch file Download diff file Expand all files Collapse all files

18
callback_plugins/logdetail.py
View file

@ -15,6 +15,10 @@
# You should have received a copy of the GNU General Public License # You should have received a copy of the GNU General Public License
# along with Ansible. If not, see <http://www.gnu.org/licenses/>. # along with Ansible. If not, see <http://www.gnu.org/licenses/>.
# Make coding more python3-ish
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type
DOCUMENTATION = r''' DOCUMENTATION = r'''
callback: logdetail callback: logdetail
callback_type: notification callback_type: notification
@ -52,6 +56,7 @@ TIME_FORMAT="%b %d %Y %H:%M:%S"
MSG_FORMAT = "%(now)s\t%(count)s\t%(category)s\t%(name)s\t%(data)s\n" MSG_FORMAT = "%(now)s\t%(count)s\t%(category)s\t%(name)s\t%(data)s\n"
def getlogin(): def getlogin():
try: try:
user = os.getlogin() user = os.getlogin()
@ -59,6 +64,7 @@ def getlogin():
user = pwd.getpwuid(os.geteuid())[0] user = pwd.getpwuid(os.geteuid())[0]
return user return user
class LogMech(object): class LogMech(object):
def __init__(self, logpath): def __init__(self, logpath):
self.started = time.time() self.started = time.time()
@ -136,7 +142,6 @@ class LogMech(object):
else: else:
name = "unknown" name = "unknown"
# we're in setup - move the invocation info up one level # we're in setup - move the invocation info up one level
if 'invocation' in data: if 'invocation' in data:
invoc = data['invocation'] invoc = data['invocation']
@ -175,7 +180,6 @@ class LogMech(object):
fd.close() fd.close()
class CallbackModule(CallbackBase): class CallbackModule(CallbackBase):
""" """
logs playbook results, per host, in /var/log/ansible/hosts logs playbook results, per host, in /var/log/ansible/hosts
@ -227,7 +231,8 @@ class CallbackModule(CallbackBase):
def v2_playbook_on_task_start(self, task, is_conditional): def v2_playbook_on_task_start(self, task, is_conditional):
self.task = task self.task = task
self.task._name = task.name if self.task:
self.task._name = task.get_name().strip()
self.logmech._last_task_start = time.time() self.logmech._last_task_start = time.time()
self._task_count += 1 self._task_count += 1
@ -264,6 +269,7 @@ class CallbackModule(CallbackBase):
pb_info['extra_vars'] = play._variable_manager.extra_vars pb_info['extra_vars'] = play._variable_manager.extra_vars
pb_info['inventory'] = play._variable_manager._inventory._sources pb_info['inventory'] = play._variable_manager._inventory._sources
pb_info['playbook_checksum'] = secure_hash(path) pb_info['playbook_checksum'] = secure_hash(path)
if hasattr(self, "play_context"):
pb_info['check'] = self.play_context.check_mode pb_info['check'] = self.play_context.check_mode
pb_info['diff'] = self.play_context.diff pb_info['diff'] = self.play_context.diff
self.logmech.play_log(json.dumps(pb_info, indent=4)) self.logmech.play_log(json.dumps(pb_info, indent=4))
@ -273,8 +279,9 @@ class CallbackModule(CallbackBase):
info = {} info = {}
info['play'] = play.name info['play'] = play.name
info['hosts'] = play.hosts info['hosts'] = play.hosts
info['transport'] = str(self.play_context.connection)
info['number'] = self._play_count info['number'] = self._play_count
if hasattr(self, "play_context"):
info['transport'] = str(self.play_context.connection)
info['check'] = self.play_context.check_mode info['check'] = self.play_context.check_mode
info['diff'] = self.play_context.diff info['diff'] = self.play_context.diff
self.logmech.play_info = info self.logmech.play_info = info
@ -283,7 +290,6 @@ class CallbackModule(CallbackBase):
except TypeError: except TypeError:
print(("Failed to conver to JSON:", info)) print(("Failed to conver to JSON:", info))
def v2_playbook_on_stats(self, stats): def v2_playbook_on_stats(self, stats):
results = {} results = {}
for host in list(stats.processed.keys()): for host in list(stats.processed.keys()):
@ -292,5 +298,3 @@ class CallbackModule(CallbackBase):
self.logmech.play_log(json.dumps({'stats': results}, indent=4)) self.logmech.play_log(json.dumps({'stats': results}, indent=4))
self.logmech.play_log(json.dumps({'playbook_end': time.time()}, indent=4)) self.logmech.play_log(json.dumps({'playbook_end': time.time()}, indent=4))
print(('logs written to: %s' % self.logmech.logpath_play)) print(('logs written to: %s' % self.logmech.logpath_play))

101
scripts/logview
View file

@ -1,16 +1,23 @@
#!/usr/bin/python3 #!/usr/bin/python
# -*- coding: utf-8 -*-
# vim: et ts=4 ai sw=4 sts=0 # vim: et ts=4 ai sw=4 sts=0
import sys import sys
import json import json
from optparse import OptionParser from argparse import ArgumentParser
import os import os
import re
import glob import glob
import gzip import gzip
from datetime import datetime, date, timedelta from datetime import datetime, timedelta
import dateutil.parser as dateparser import dateutil.parser as dateparser
try:
# Python3
import configparser import configparser
except ImportError:
# Python2
import ConfigParser as configparser
from ansible.config.manager import find_ini_config_file from ansible.config.manager import find_ini_config_file
from ansible.utils.color import hostcolor, stringc from ansible.utils.color import stringc
from ansible import constants as C from ansible import constants as C
from collections import Counter from collections import Counter
@ -19,11 +26,13 @@ if not hasattr(sys.stdout, 'isatty') or not sys.stdout.isatty():
else: else:
HAS_COLOR = True HAS_COLOR = True
logpath = '/var/log/ansible' DEFAULT_LOGPATH = '/var/log/ansible'
default_search_terms = ['CHANGED', 'FAILED'] default_search_terms = ['CHANGED', 'FAILED']
date_terms = { date_terms = {
"today": lambda: datetime.today().replace(hour=0, minute=0, second=0, microsecond=0), "today": lambda: datetime.today().replace(
"yesterday": lambda: datetime.today().replace(hour=0, minute=0, second=0, microsecond=0) - timedelta(1), hour=0, minute=0, second=0, microsecond=0),
"yesterday": lambda: datetime.today().replace(
hour=0, minute=0, second=0, microsecond=0) - timedelta(1),
} }
@ -58,12 +67,14 @@ def colorByStats(txt, stats):
else: else:
return stringc(txt, C.COLOR_OK) return stringc(txt, C.COLOR_OK)
def colorByCount(txt, count, color): def colorByCount(txt, count, color):
s = "%s%s" % (txt, count) s = "%s%s" % (txt, count)
if count > 0 and HAS_COLOR: if count > 0 and HAS_COLOR:
s = stringc(s, color) s = stringc(s, color)
return s return s
def parse_info(infofile): def parse_info(infofile):
data = {} data = {}
with open(infofile) as f: with open(infofile) as f:
@ -87,6 +98,7 @@ def format_stats(stats):
colorByCount("unr:", stats['unreachable'], C.COLOR_UNREACHABLE), colorByCount("unr:", stats['unreachable'], C.COLOR_UNREACHABLE),
colorByCount("fail:", stats['failures'], C.COLOR_ERROR)) colorByCount("fail:", stats['failures'], C.COLOR_ERROR))
def col_width(rows): def col_width(rows):
widths = [] widths = []
for col in zip(*(rows)): for col in zip(*(rows)):
@ -95,11 +107,17 @@ def col_width(rows):
widths[-1] = 0 # don't pad last column widths[-1] = 0 # don't pad last column
return widths return widths
def date_cheat(datestr): def date_cheat(datestr):
dc = date_terms.get(datestr, lambda: dateparser.parse(datestr)) dc = date_terms.get(datestr, lambda: dateparser.parse(datestr))
return dc() return dc()
def date_from_path(path):
date_comp = re.search(r'/(\d{4})/(\d{2})/(\d{2})', path)
return datetime(*map(int, date_comp.groups()))
def parse_args(args): def parse_args(args):
usage = """ usage = """
logview [options] [-d datestr] [-p playbook] logview [options] [-d datestr] [-p playbook]
@ -114,28 +132,31 @@ def parse_args(args):
logview -s ANY -d yesterday -p mirrorlist # list all events from the mirrorlist playbook logview -s ANY -d yesterday -p mirrorlist # list all events from the mirrorlist playbook
""" """
parser = OptionParser(usage=usage) parser = ArgumentParser(usage=usage)
parser.add_option("-d", default='today', dest='datestr', help="time string of when you want logs") date_group = parser.add_mutually_exclusive_group()
parser.add_option("-p", default='*', dest='playbook', help="the playbook you want to look for") date_group.add_argument("-d", default='today', dest='datestr', help="display logs from specified date")
parser.add_option("-H", default=[], dest='hostname', action='append', help="Limit to the specified hostname") date_group.add_argument("--since", dest="since", help="display logs since specified date")
parser.add_option("-m", default=False, dest='message', action='store_true', help='Show tasks output') date_group.add_argument("--all", default=False, dest="list_all", action='store_true', help="display all logs")
parser.add_option("-v", default=False, dest='verbose', action='store_true', help='Verbose') parser.add_argument("-p", default='*', dest='playbook', help="the playbook you want to look for")
parser.add_option("-s", default=[], dest='search_terms', action='append', help="status to search for") parser.add_argument("-H", default=[], dest='hostname', action='append', help="Limit to the specified hostname")
parser.add_option("-l", default=False, dest="list_pb", action='store_true', help="list playbooks for a specific date") parser.add_argument("-m", default=False, dest='message', action='store_true', help='Show tasks output')
parser.add_option("-L", default=False, dest="list_all_pb", action='store_true', help="list all playbooks ever ran") parser.add_argument("-v", default=False, dest='verbose', action='store_true', help='Verbose')
parser.add_option("--profile", default=False, dest="profile", action='store_true', help="output timing input per task") parser.add_argument("-s", default=[], dest='search_terms', action='append', help="status to search for")
(opts, args) = parser.parse_args(args) parser.add_argument("-l", default=False, dest="list_pb", action='store_true', help="list playbook runs")
parser.add_argument("--profile", default=False, dest="profile", action='store_true', help="output timing input per task")
opts = parser.parse_args(args)
opts.datestr = date_cheat(opts.datestr) opts.datestr = date_cheat(opts.datestr)
if not opts.search_terms: if not opts.search_terms:
opts.search_terms = default_search_terms opts.search_terms = default_search_terms
if opts.since:
opts.since = date_cheat(opts.since)
opts.search_terms = list(map(str.upper, opts.search_terms)) opts.search_terms = list(map(str.upper, opts.search_terms))
return opts, args return opts
def search_logs(opts, logfiles): def search_logs(opts, logfiles):
rows = [] rows = []
headers = []
msg = '' msg = ''
for fn in sorted(logfiles): for fn in sorted(logfiles):
hostname = os.path.basename(fn).replace('.log', '').replace('.gz', '') hostname = os.path.basename(fn).replace('.log', '').replace('.gz', '')
@ -148,13 +169,13 @@ def search_logs(opts, logfiles):
with gzip.open(fn) as f: with gzip.open(fn) as f:
f.read() f.read()
open_f = gzip.open(fn, "rt") open_f = gzip.open(fn, "rt")
except: except IOError:
open_f = open(fn) open_f = open(fn)
for line in open_f: for line in open_f:
things = line.split('\t') things = line.split('\t')
if len(things) < 5: if len(things) < 5:
print("(logview error - unhandled line): %r\n" % line) msg += "(logview error - unhandled line): %r\n" % line
continue continue
# See callback_plugins/logdetail.py for how these lines get created. # See callback_plugins/logdetail.py for how these lines get created.
@ -180,11 +201,14 @@ def search_logs(opts, logfiles):
c_hostname = colorByStats(hostname, slurp) c_hostname = colorByStats(hostname, slurp)
state = colorByStats(category, slurp) state = colorByStats(category, slurp)
result = [timestamp, c_hostname, task_ts, count, state] result = [timestamp, c_hostname, task_ts, count, state]
if not name: if not name:
name = slurp.get("task_module") name = slurp.get("task_module")
try:
name = name.decode('utf8')
except AttributeError:
pass
result.append(name) result.append(name)
if dur: if dur:
@ -220,21 +244,26 @@ def main(args):
if cfg: if cfg:
cp = configparser.ConfigParser() cp = configparser.ConfigParser()
cp.read(cfg) cp.read(cfg)
logpath = cp.get('callback_logdetail', "log_path", fallback="/var/log/ansible") try:
opts, args = parse_args(args) logpath = cp.get('callback_logdetail', "log_path")
except configparser.NoSectionError:
logpath = DEFAULT_LOGPATH
opts = parse_args(args)
rows = [] rows = []
# List play summary # List play summary
if opts.list_pb or opts.list_all_pb: if opts.list_pb:
rows.append(["Date", colorByCat("", "Playbook"), "Ran By", "Hosts", "Stats"]) rows.append(["Date", colorByCat("", "Playbook"), "Ran By", "Hosts", "Stats"])
for r, d, f in os.walk(logpath): for r, d, f in os.walk(logpath):
if opts.since and f and date_from_path(r) < opts.since:
continue
for file in f: for file in f:
if file.endswith('.info'): if file.endswith('.info'):
pb = parse_info(os.path.join(r, file)) pb = parse_info(os.path.join(r, file))
pb_name = os.path.splitext(os.path.basename(pb['playbook']))[0] pb_name = os.path.splitext(os.path.basename(pb['playbook']))[0]
pb_date = datetime.fromtimestamp(pb['playbook_start']) pb_date = datetime.fromtimestamp(pb['playbook_start'])
if ( if (
opts.list_all_pb opts.list_all or opts.since
or ( or (
opts.datestr != opts.datestr.replace(hour=0, minute=0, second=0, microsecond=0) opts.datestr != opts.datestr.replace(hour=0, minute=0, second=0, microsecond=0)
and opts.datestr == pb_date) and opts.datestr == pb_date)
@ -253,7 +282,6 @@ def main(args):
summary = format_stats(stats) summary = format_stats(stats)
# summary = "ok:%s chd:%s unr:%s faild:%s" % (stats['ok'], stats['changed'], stats['unreachable'], stats['failures']) # summary = "ok:%s chd:%s unr:%s faild:%s" % (stats['ok'], stats['changed'], stats['unreachable'], stats['failures'])
rows.append([str(pb_date), pb_name, pb['userid'], str(host_count), summary]) rows.append([str(pb_date), pb_name, pb['userid'], str(host_count), summary])
m_widths = col_width(rows) m_widths = col_width(rows)
@ -263,12 +291,19 @@ def main(args):
for row in rows: for row in rows:
print(" ".join((val.ljust(width) for val, width in zip(row, m_widths))).strip()) print(" ".join((val.ljust(width) for val, width in zip(row, m_widths))).strip())
# Play detail # Play detail
else: else:
for pb in glob.glob(os.path.join(logpath, opts.playbook)): for pb in glob.glob(os.path.join(logpath, opts.playbook)):
pb_name = os.path.basename(pb) pb_name = os.path.basename(pb)
for pb_logdir in glob.glob(os.path.join(pb, opts.datestr.strftime("%Y/%m/%d"))): if opts.list_all or opts.since:
date_glob = glob.glob(os.path.join(pb, "*/*/*"))
else:
date_glob = glob.glob(os.path.join(pb, opts.datestr.strftime("%Y/%m/%d")))
for pb_logdir in date_glob:
if opts.since:
run_date = date_from_path(pb_logdir)
if run_date < opts.since:
continue
if opts.datestr != opts.datestr.replace(hour=0, minute=0, second=0, microsecond=0): if opts.datestr != opts.datestr.replace(hour=0, minute=0, second=0, microsecond=0):
logfiles = glob.glob(pb_logdir + '/' + opts.datestr.strftime("%H.%M.%S") + '/*.log*') logfiles = glob.glob(pb_logdir + '/' + opts.datestr.strftime("%H.%M.%S") + '/*.log*')
else: else:
@ -276,11 +311,11 @@ def main(args):
rows = search_logs(opts, logfiles) rows = search_logs(opts, logfiles)
if rows: if rows:
m_widths = col_width(rows) m_widths = col_width(rows)
print(pb_name) print("%s\n-------" % (pb_name,))
for row in rows: for row in rows:
print(" ".join((val.ljust(width) for val, width in zip(row, m_widths)))) print(" ".join((val.ljust(width) for val, width in zip(row, m_widths))))
#print(pb_name) print("")
#print(msg)
if __name__ == "__main__": if __name__ == "__main__":
sys.exit(main(sys.argv[1:])) sys.exit(main(sys.argv[1:]))