Infrastructure/ansible
1
0
Fork 0
Code Issues Pull requests 6 Projects Releases Packages Wiki Activity Actions

Update the badges cronjob to use FASJSON or LDAP

Browse source 
Signed-off-by: Aurélien Bompard <aurelien@bompard.org>
This commit is contained in:
Aurélien Bompard 2022-06-28 18:41:52 +02:00 • committed by kevin
parent 5d1c34bb1b
commit 7cc09628a5
4 changed files with 191 additions and 227 deletions
Download patch file Download diff file Expand all files Collapse all files

67
roles/badges/backend/files/cron/award-badges-dev-badge
View file

@ -12,9 +12,13 @@ import os
import subprocess import subprocess
import shutil import shutil
from gssapi import Credentials
from gssapi.exceptions import GSSError
from requests_gssapi import HTTPSPNEGOAuth
from tahrir_api.dbapi import TahrirDatabase from tahrir_api.dbapi import TahrirDatabase
import fedbadges.utils import fedbadges.utils
import transaction import transaction
import requests
import fedora.client import fedora.client
import socket import socket
@ -34,33 +38,19 @@ fedmsg.init(**fm_config)
badge = None badge = None
email_to_username_mapping = {} http_client = None
_fas_cache = {}
def make_email_to_username_mapping(fas_credentials): def get_http_client():
fasclient = fedora.client.fas2.AccountSystem( os.environ["KRB5_CLIENT_KTNAME"] = fm_config.get("keytab")
username=fas_credentials['username'],
password=fas_credentials['password'],
)
timeout = socket.getdefaulttimeout()
socket.setdefaulttimeout(600)
try: try:
log.info("Downloading FAS cache") creds = Credentials(usage="initiate")
request = fasclient.send_request('/user/list', except GSSError as e:
req_params={'search': '*'}, log.error("GSSError trying to authenticate with Kerberos", e)
auth=True) gssapi_auth = HTTPSPNEGOAuth(opportunistic_auth=True, creds=creds)
finally: session = requests.Session()
socket.setdefaulttimeout(timeout) session.auth = gssapi_auth
return session
results = {}
for person in request['people']:
if person.email:
results[person.email] = person.username
return results
def clone_repo(repo): def clone_repo(repo):
@ -88,26 +78,20 @@ def gather_authors(location):
os.chdir(pwd) os.chdir(pwd)
return authors return authors
def emails_to_fas_accounts(emails): def emails_to_fas_accounts(emails):
usernames = [] usernames = []
for email in emails: for email in emails:
if email.endswith('@fedoraproject.org'): result = http_client.get(
username = email[:-1 * len('@fedoraproject.org')] "{}search/users/".format(fm_config['fasjson_base_url']),
if not username in usernames: params={"email": email}
usernames.append(username) )
if not result.ok:
continue continue
response = result.json()
# Otherwise, look it up in FAS and append if response["page"]["total_results"] != 1:
raise WTFError("wtf is going on with fas") continue
usernames.append(response["result"][0]["username"])
return usernames
def emails_to_fas_accounts(emails):
usernames = []
for email in emails:
if email in email_to_username_mapping:
usernames.append(email_to_username_mapping[email])
return usernames return usernames
@ -153,6 +137,5 @@ if __name__ == '__main__':
badge = tahrir.get_badge(badge_id='badge-off!') badge = tahrir.get_badge(badge_id='badge-off!')
if not badge: if not badge:
raise ValueError("badge does not exist") raise ValueError("badge does not exist")
email_to_username_mapping = make_email_to_username_mapping( http_client = get_http_client()
fm_config['fas_credentials'])
main() main()

100
roles/badges/backend/files/cron/award-flock-paparazzi-badge
View file

@ -11,19 +11,20 @@ Author: Ralph Bean <rbean@redhat.com>
from __future__ import print_function from __future__ import print_function
import __main__ import __main__
__main__.__requires__ = __requires__ = ["tahrir-api", "sqlalchemy>=0.7"]; __main__.__requires__ = __requires__ = ["tahrir-api", "sqlalchemy>=0.7"]
import pkg_resources import pkg_resources
pkg_resources.require(__requires__) pkg_resources.require(__requires__)
import socket
import time import time
import getpass import os
import ConfigParser import ConfigParser
import requests import requests
import fedora.client
import transaction import transaction
import tahrir_api.dbapi import tahrir_api.dbapi
from gssapi import Credentials
from gssapi.exceptions import GSSError
from requests_gssapi import HTTPSPNEGOAuth
import fedmsg import fedmsg
import fedmsg.config import fedmsg.config
@ -43,8 +44,6 @@ config.read(['flock-paparazzi.ini', '/etc/flock-paparazzi.ini'])
flickr_api_key = config.get('general', 'flickr_api_key') flickr_api_key = config.get('general', 'flickr_api_key')
g_plus_key = config.get('general', 'g_plus_key') g_plus_key = config.get('general', 'g_plus_key')
userIP = config.get('general', 'userIP') userIP = config.get('general', 'userIP')
fas_username = config.get('general', 'fas_username')
fas_password = config.get('general', 'fas_password')
# API urls # API urls
flickr_url = 'https://api.flickr.com/services/rest/' flickr_url = 'https://api.flickr.com/services/rest/'
@ -52,7 +51,19 @@ g_plus_url = 'https://www.googleapis.com/plus/v1/activities'
badge_id = 'flock-paparazzi' badge_id = 'flock-paparazzi'
_fas_cache = {} http_client = None
def get_http_client():
os.environ["KRB5_CLIENT_KTNAME"] = fm_config.get("keytab")
try:
creds = Credentials(usage="initiate")
except GSSError as e:
print("GSSError trying to authenticate with Kerberos", e)
gssapi_auth = HTTPSPNEGOAuth(opportunistic_auth=True, creds=creds)
session = requests.Session()
session.auth = gssapi_auth
return session
def get_g_plus_persons(query): def get_g_plus_persons(query):
@ -148,50 +159,24 @@ def get_flickr_persons(tags):
yield value yield value
def make_fas_cache(username, password): def get_username(name):
global _fas_cache # First, check if the same username exists:
if _fas_cache: response = http_client.get(
return _fas_cache "%susers/%s/" % (fm_config['fasjson_base_url'], name)
print("No previous fas cache found. Looking to rebuild.")
try:
import fedora.client.fas2
except ImportError:
print("No python-fedora installed. Not caching fas.")
return {}
if not username or not password:
print("No fas credentials found. Not caching fas.")
return {}
fasclient = fedora.client.fas2.AccountSystem(
username=username,
password=password,
) )
if response.ok:
timeout = socket.getdefaulttimeout() return name
socket.setdefaulttimeout(600) # Now try with the human name
try: response = http_client.get(
print("Downloading FAS cache...") "%ssearch/users/" % fm_config['fasjson_base_url'],
request = fasclient.send_request( params={"human_name": name}
'/user/list',
req_params={'search': '*'},
auth=True,
) )
finally: if not response.ok:
socket.setdefaulttimeout(timeout) return None
response = response.json()
print("Caching necessary user data") if response["page"]["total_results"] != 1:
for user in request['people']: return None
for key in ['username', 'human_name']: return response["result"][0]["username"]
if user[key]:
_fas_cache[user[key]] = user['username']
del request
del fasclient
return _fas_cache
def get_persons(): def get_persons():
@ -206,6 +191,7 @@ def get_persons():
def main(): def main():
global http_client
# First, initialize the tahrir db connection # First, initialize the tahrir db connection
uri = fm_config['badges_global']['database_uri'] uri = fm_config['badges_global']['database_uri']
tahrir = tahrir_api.dbapi.TahrirDatabase( tahrir = tahrir_api.dbapi.TahrirDatabase(
@ -213,8 +199,7 @@ def main():
notification_callback=fedbadges.utils.notification_callback, notification_callback=fedbadges.utils.notification_callback,
) )
# Then, build a fas cache. this takes forever.. http_client = get_http_client()
cache = make_fas_cache(fas_username, fas_password)
badge = tahrir.get_badge(badge_id) badge = tahrir.get_badge(badge_id)
already_has_it = [a.person.nickname for a in badge.assertions] already_has_it = [a.person.nickname for a in badge.assertions]
@ -231,14 +216,15 @@ def main():
continue continue
print("* Considering", person) print("* Considering", person)
if person in cache: username = get_username(person)
if cache[person] in already_has_it: if username is not None:
print("Skipping %r" % cache[person]) if username in already_has_it:
print("Skipping %r" % username)
continue continue
print(" *", cache[person], "gets the badge") print(" *", username, "gets the badge")
already_has_it.append(cache[person]) already_has_it.append(username)
email = cache[person] + "@fedoraproject.org" email = username + "@fedoraproject.org"
try: try:
transaction.begin() transaction.begin()
tahrir.add_assertion(badge_id, email, None) tahrir.add_assertion(badge_id, email, None)

119
roles/badges/backend/files/cron/award-lifecycle-badges
View file

@ -6,20 +6,19 @@ __main__.__requires__ = __requires__ = ["tahrir-api", "sqlalchemy>=0.7"];
import pkg_resources import pkg_resources
pkg_resources.require(__requires__) pkg_resources.require(__requires__)
import os
import datetime import datetime
import time import time
import urllib import urllib
import socket import socket
import logging
import re
from tahrir_api.dbapi import TahrirDatabase from tahrir_api.dbapi import TahrirDatabase
import transaction import transaction
_fas_cache = {}
import logging
log = logging.getLogger() log = logging.getLogger()
logging.basicConfig() logging.basicConfig()
import fedora.client.fas2
import fedmsg import fedmsg
import fedmsg.config import fedmsg.config
@ -32,39 +31,83 @@ fedmsg.init(**fm_config)
import fedbadges.utils import fedbadges.utils
# generates a list of search terms import ldap
# alpha map is just a lowercase english alphabet import ldap.sasl
from ldap.controls.libldap import SimplePagedResultsControl
def gen_fas_searchterms(): LDAP_CONF = "/etc/openldap/ldap.conf"
alpha = map(chr, range(97, 123))
searchterms = [ alpha_ltr + "*" for alpha_ltr in alpha ]
return searchterms
def get_fas_userlist(fas_credentials, search_qry): class LDAPClient(object):
creds = fas_credentials
fasclient = fedora.client.fas2.AccountSystem( def __init__(self):
username=creds['username'], self.config = {}
password=creds['password'], self.conn = None
self._read_config()
def _read_config(self):
conf_re = re.compile(r"^([A-Z_]+)\s+(.+)$")
with open(LDAP_CONF) as cf:
for line in cf:
mo = conf_re.match(line.strip())
if mo is None:
continue
variable = mo.group(1)
value = mo.group(2)
self.config[variable] = value
def connect(self):
self.conn = ldap.initialize(self.config["URI"].split(" ")[0])
self.conn.protocol_version = 3
self.conn.sasl_interactive_bind_s('', ldap.sasl.gssapi())
def search(self, base, filters, attrs):
page_size = 1000
base_dn = "{base},{main_base}".format(base=base,main_base=self.config['BASE'])
page_cookie = ""
while True:
page_control = SimplePagedResultsControl(
criticality=False, size=page_size, cookie=page_cookie
) )
msgid = self.conn.search_ext(
base_dn,
ldap.SCOPE_SUBTREE,
filters,
attrlist=attrs,
serverctrls=[page_control],
)
rtype, rdata, rmsgid, serverctrls = self.conn.result3(msgid)
for dn, obj in rdata:
yield obj
for ctrl in serverctrls:
if isinstance(ctrl, SimplePagedResultsControl):
page_cookie = ctrl.cookie
break
if not page_cookie:
break
timeout = socket.getdefaulttimeout()
socket.setdefaulttimeout(600)
try:
log.info("Downloading FAS cache")
request = fasclient.send_request('/user/list',
req_params={'search': search_qry},
auth=True)
finally: def get_fas_userlist(threshold):
socket.setdefaulttimeout(timeout) os.environ["KRB5_CLIENT_KTNAME"] = fm_config.get("keytab")
ldap_client = LDAPClient()
# We don't actually check for CLA+1, just "2 groups" ldap_client.connect()
return [p for p in request['people'] if len(p.memberships) > 1] filters = "(&(fasCreationTime<={})(objectclass=fasUser))".format(threshold.strftime("%Y%m%d%H%M%SZ"))
response = ldap_client.search(base="cn=users,cn=accounts", filters=filters, attrs=["uid", "memberof"])
for res in response:
groups = []
for groupdn in res.get("memberof", []):
groupdn = groupdn.decode("ascii")
if not groupdn.endswith(",cn=groups,cn=accounts,{}".format(ldap_client.config['BASE'])):
continue
groupname = groupdn.split(",")[0].split("=")[1]
if groupname == "ipausers":
continue # Assume all groups are FAS groups except this one
groups.append(groupname)
yield {"username": res["uid"][0].decode("ascii"), "groups": groups}
def main(): def main():
global http_client
now = datetime.datetime.utcnow() now = datetime.datetime.utcnow()
year = datetime.timedelta(days=365.5) year = datetime.timedelta(days=365.5)
mapping = { mapping = {
@ -81,26 +124,18 @@ def main():
badge = tahrir.get_badge(badge_id=badge_id) badge = tahrir.get_badge(badge_id=badge_id)
assert(badge.id) assert(badge.id)
# Then, do a long query against FAS for our candidates. # Then, query IPA for users created before the threshold
# Here I call search terms to generate a lists of search terms
# Looping over the list of search terms, pass the search term to get_fas_userlists
fas_credentials = fm_config['fas_credentials']
searchterms = gen_fas_searchterms()
for search_elem in searchterms:
results = get_fas_userlist(fas_credentials, search_elem)
for badge_id, delta in mapping.items(): for badge_id, delta in mapping.items():
badge = tahrir.get_badge(badge_id=badge_id) badge = tahrir.get_badge(badge_id=badge_id)
for person in results: threshold = now - delta
creation = datetime.datetime.strptime( for person in get_fas_userlist(threshold):
person.creation[:19], '%Y-%m-%d %H:%M:%S') if len(person["groups"]) < 2:
if now - creation > delta: continue
hit_em_up(badge, person) hit_em_up(badge, person)
def hit_em_up(badge, fas_user): def hit_em_up(badge, fas_user):
email = fas_user.username + "@fedoraproject.org" email = fas_user["username"] + "@fedoraproject.org"
user = tahrir.get_person(email) user = tahrir.get_person(email)
if not user: if not user:
@ -110,7 +145,7 @@ def hit_em_up(badge, fas_user):
print email, "already has", badge.id, "skipping." print email, "already has", badge.id, "skipping."
return return
time.sleep(15) time.sleep(1)
print "awarding", badge.id, "to", email print "awarding", badge.id, "to", email
try: try:
transaction.begin() transaction.begin()

122
roles/badges/backend/files/cron/award-oldschool-badges
View file

@ -6,6 +6,7 @@ __main__.__requires__ = __requires__ = ["tahrir-api", "sqlalchemy>=0.7"];
import pkg_resources import pkg_resources
pkg_resources.require(__requires__) pkg_resources.require(__requires__)
import os
import itertools import itertools
import string import string
import time import time
@ -14,11 +15,14 @@ import socket
from hashlib import md5 from hashlib import md5
import getpass import getpass
import pprint import pprint
from collections import defaultdict
from gssapi import Credentials
from gssapi.exceptions import GSSError
from requests_gssapi import HTTPSPNEGOAuth
from tahrir_api.dbapi import TahrirDatabase from tahrir_api.dbapi import TahrirDatabase
import transaction import transaction
import requests
_fas_cache = {}
import logging import logging
log = logging.getLogger() log = logging.getLogger()
@ -37,88 +41,44 @@ fedmsg.init(**fm_config)
import fedbadges.utils import fedbadges.utils
def user_in_group(user, group_name): def get_http_client():
# First, bail out if they're not in the group at all os.environ["KRB5_CLIENT_KTNAME"] = fm_config.get("keytab")
if not any([g.name == group_name for g in user.memberships]):
return False
# Find the group_id of the group we're looking for..
group_id = None
for g in user.memberships:
if g.name == group_name:
group_id = g.id
break
if not group_id:
return False
# For that group_id, find the relevant role
relevant_role = None
for role in user.roles:
if role.group_id == group_id:
relevant_role = role
break
if not relevant_role:
return False
# They must be actually 'approved' in that group for this to count
return relevant_role.role_status == 'approved'
def get_all_fas_users(creds):
fasclient = fedora.client.fas2.AccountSystem(
username=creds['username'],
password=creds['password'],
)
timeout = socket.getdefaulttimeout()
socket.setdefaulttimeout(600)
try: try:
log.info("Downloading FAS cache") creds = Credentials(usage="initiate")
for a, b in itertools.product(string.lowercase, string.lowercase): except GSSError as e:
term = a + b + '*' log.error("GSSError trying to authenticate with Kerberos", e)
log.info(" Querying FAS for %r" % term) gssapi_auth = HTTPSPNEGOAuth(opportunistic_auth=True, creds=creds)
request = fasclient.send_request('/user/list', session = requests.Session()
req_params={'search': term}, session.auth = gssapi_auth
auth=True) return session
log.info(" Found %r matching users." % len(request['people']))
for person in request['people']:
yield person
finally:
socket.setdefaulttimeout(timeout)
def get_fas_groupings(fas_credentials, lookup, **config): def get_fas_groupings(fas_credentials, lookup, **config):
results = {} results = defaultdict(list)
packager_id, ambassadors_id = None, None membership_types = ("members", "sponsors")
sponsor_types = ['sponsor', 'administrator'] http_client = get_http_client()
mega_list = get_all_fas_users(fas_credentials)
for user in mega_list:
# This is the main check.
for group_name, badge_id in lookup.iteritems(): for group_name, badge_id in lookup.iteritems():
if user_in_group(user, group_name): # This is the main check.
results[group_name] = results.get(group_name, []) + [user] for membership_type in membership_types:
url = "%sgroups/%s/%s/" % (
fm_config['fasjson_base_url'],
group_name,
membership_type
)
response = http_client.get(url)
if not response.ok:
continue
for user in response.json()["result"]:
username = user["username"]
results[group_name].append(username)
# Beyond the main check, here is a special check that makes sure they # Beyond the main check, here is a special check for the sponsors of the packager and ambassadors groups
# are a sponsor in the packager group. if membership_type != "sponsors":
if not packager_id: continue
for group in user.memberships: if group_name == "packager":
if group.name == 'packager': results["sponsors"].append(username)
packager_id = group.id elif group_name == "ambassadors":
results["ambassadors_sponsors"].append(username)
if not ambassadors_id:
for group in user.memberships:
if group.name == 'ambassadors':
ambassadors_id = group.id
for role in user.roles:
if role.group_id == packager_id:
if role.role_type in sponsor_types and role.role_status == 'approved':
results['sponsors'] = results.get('sponsors', []) + [user]
if role.group_id == ambassadors_id:
if role.role_type in sponsor_types and role.role_status == 'approved':
results['ambassadors_sponsors'] = results.get('ambassadors_sponsors', []) + [user]
return results return results
@ -179,9 +139,9 @@ def main():
hit_em_up(badge, members) hit_em_up(badge, members)
def hit_em_up(badge, group): def hit_em_up(badge, members):
for fas_user in group: for username in members:
email = fas_user.username + "@fedoraproject.org" email = username + "@fedoraproject.org"
user = tahrir.get_person(email) user = tahrir.get_person(email)
if not user: if not user: