Infrastructure/ansible
1
0
Fork 0
Code Issues Pull requests 6 Projects Releases Packages Wiki Activity Actions

Update the badges cronjob to use FASJSON or LDAP

Browse source 
Signed-off-by: Aurélien Bompard <aurelien@bompard.org>
This commit is contained in:
Aurélien Bompard 2022-06-28 18:41:52 +02:00 • committed by kevin
parent 5d1c34bb1b
commit 7cc09628a5
4 changed files with 191 additions and 227 deletions
Download patch file Download diff file Expand all files Collapse all files

67
roles/badges/backend/files/cron/award-badges-dev-badge
View file

@ -12,9 +12,13 @@ import os
import subprocess
import shutil
from gssapi import Credentials
from gssapi.exceptions import GSSError
from requests_gssapi import HTTPSPNEGOAuth
from tahrir_api.dbapi import TahrirDatabase
import fedbadges.utils
import transaction
import requests
import fedora.client
import socket
@ -34,33 +38,19 @@ fedmsg.init(**fm_config)
badge = None
email_to_username_mapping = {}
_fas_cache = {}
http_client = None
def make_email_to_username_mapping(fas_credentials):
fasclient = fedora.client.fas2.AccountSystem(
username=fas_credentials['username'],
password=fas_credentials['password'],
)
timeout = socket.getdefaulttimeout()
socket.setdefaulttimeout(600)
def get_http_client():
os.environ["KRB5_CLIENT_KTNAME"] = fm_config.get("keytab")
try:
log.info("Downloading FAS cache")
request = fasclient.send_request('/user/list',
req_params={'search': '*'},
auth=True)
finally:
socket.setdefaulttimeout(timeout)
results = {}
for person in request['people']:
if person.email:
results[person.email] = person.username
return results
creds = Credentials(usage="initiate")
except GSSError as e:
log.error("GSSError trying to authenticate with Kerberos", e)
gssapi_auth = HTTPSPNEGOAuth(opportunistic_auth=True, creds=creds)
session = requests.Session()
session.auth = gssapi_auth
return session
def clone_repo(repo):
@ -88,26 +78,20 @@ def gather_authors(location):
os.chdir(pwd)
return authors
def emails_to_fas_accounts(emails):
usernames = []
for email in emails:
if email.endswith('@fedoraproject.org'):
username = email[:-1 * len('@fedoraproject.org')]
if not username in usernames:
usernames.append(username)
result = http_client.get(
"{}search/users/".format(fm_config['fasjson_base_url']),
params={"email": email}
)
if not result.ok:
continue
# Otherwise, look it up in FAS and append
raise WTFError("wtf is going on with fas")
return usernames
def emails_to_fas_accounts(emails):
usernames = []
for email in emails:
if email in email_to_username_mapping:
usernames.append(email_to_username_mapping[email])
response = result.json()
if response["page"]["total_results"] != 1:
continue
usernames.append(response["result"][0]["username"])
return usernames
@ -153,6 +137,5 @@ if __name__ == '__main__':
badge = tahrir.get_badge(badge_id='badge-off!')
if not badge:
raise ValueError("badge does not exist")
email_to_username_mapping = make_email_to_username_mapping(
fm_config['fas_credentials'])
http_client = get_http_client()
main()

102
roles/badges/backend/files/cron/award-flock-paparazzi-badge
View file

@ -11,19 +11,20 @@ Author: Ralph Bean <rbean@redhat.com>
from __future__ import print_function
import __main__
__main__.__requires__ = __requires__ = ["tahrir-api", "sqlalchemy>=0.7"];
__main__.__requires__ = __requires__ = ["tahrir-api", "sqlalchemy>=0.7"]
import pkg_resources
pkg_resources.require(__requires__)
import socket
import time
import getpass
import os
import ConfigParser
import requests
import fedora.client
import transaction
import tahrir_api.dbapi
from gssapi import Credentials
from gssapi.exceptions import GSSError
from requests_gssapi import HTTPSPNEGOAuth
import fedmsg
import fedmsg.config
@ -43,8 +44,6 @@ config.read(['flock-paparazzi.ini', '/etc/flock-paparazzi.ini'])
flickr_api_key = config.get('general', 'flickr_api_key')
g_plus_key = config.get('general', 'g_plus_key')
userIP = config.get('general', 'userIP')
fas_username = config.get('general', 'fas_username')
fas_password = config.get('general', 'fas_password')
# API urls
flickr_url = 'https://api.flickr.com/services/rest/'
@ -52,7 +51,19 @@ g_plus_url = 'https://www.googleapis.com/plus/v1/activities'
badge_id = 'flock-paparazzi'
_fas_cache = {}
http_client = None
def get_http_client():
os.environ["KRB5_CLIENT_KTNAME"] = fm_config.get("keytab")
try:
creds = Credentials(usage="initiate")
except GSSError as e:
print("GSSError trying to authenticate with Kerberos", e)
gssapi_auth = HTTPSPNEGOAuth(opportunistic_auth=True, creds=creds)
session = requests.Session()
session.auth = gssapi_auth
return session
def get_g_plus_persons(query):
@ -148,50 +159,24 @@ def get_flickr_persons(tags):
yield value
def make_fas_cache(username, password):
global _fas_cache
if _fas_cache:
return _fas_cache
print("No previous fas cache found. Looking to rebuild.")
try:
import fedora.client.fas2
except ImportError:
print("No python-fedora installed. Not caching fas.")
return {}
if not username or not password:
print("No fas credentials found. Not caching fas.")
return {}
fasclient = fedora.client.fas2.AccountSystem(
username=username,
password=password,
def get_username(name):
# First, check if the same username exists:
response = http_client.get(
"%susers/%s/" % (fm_config['fasjson_base_url'], name)
)
timeout = socket.getdefaulttimeout()
socket.setdefaulttimeout(600)
try:
print("Downloading FAS cache...")
request = fasclient.send_request(
'/user/list',
req_params={'search': '*'},
auth=True,
)
finally:
socket.setdefaulttimeout(timeout)
print("Caching necessary user data")
for user in request['people']:
for key in ['username', 'human_name']:
if user[key]:
_fas_cache[user[key]] = user['username']
del request
del fasclient
return _fas_cache
if response.ok:
return name
# Now try with the human name
response = http_client.get(
"%ssearch/users/" % fm_config['fasjson_base_url'],
params={"human_name": name}
)
if not response.ok:
return None
response = response.json()
if response["page"]["total_results"] != 1:
return None
return response["result"][0]["username"]
def get_persons():
@ -206,6 +191,7 @@ def get_persons():
def main():
global http_client
# First, initialize the tahrir db connection
uri = fm_config['badges_global']['database_uri']
tahrir = tahrir_api.dbapi.TahrirDatabase(
@ -213,8 +199,7 @@ def main():
notification_callback=fedbadges.utils.notification_callback,
)
# Then, build a fas cache. this takes forever..
cache = make_fas_cache(fas_username, fas_password)
http_client = get_http_client()
badge = tahrir.get_badge(badge_id)
already_has_it = [a.person.nickname for a in badge.assertions]
@ -231,14 +216,15 @@ def main():
continue
print("* Considering", person)
if person in cache:
if cache[person] in already_has_it:
print("Skipping %r" % cache[person])
username = get_username(person)
if username is not None:
if username in already_has_it:
print("Skipping %r" % username)
continue
print(" *", cache[person], "gets the badge")
already_has_it.append(cache[person])
email = cache[person] + "@fedoraproject.org"
print(" *", username, "gets the badge")
already_has_it.append(username)
email = username + "@fedoraproject.org"
try:
transaction.begin()
tahrir.add_assertion(badge_id, email, None)

127
roles/badges/backend/files/cron/award-lifecycle-badges
View file

@ -6,20 +6,19 @@ __main__.__requires__ = __requires__ = ["tahrir-api", "sqlalchemy>=0.7"];
import pkg_resources
pkg_resources.require(__requires__)
import os
import datetime
import time
import urllib
import socket
import logging
import re
from tahrir_api.dbapi import TahrirDatabase
import transaction
_fas_cache = {}
import logging
log = logging.getLogger()
logging.basicConfig()
import fedora.client.fas2
import fedmsg
import fedmsg.config
@ -32,39 +31,83 @@ fedmsg.init(**fm_config)
import fedbadges.utils
# generates a list of search terms
# alpha map is just a lowercase english alphabet
import ldap
import ldap.sasl
from ldap.controls.libldap import SimplePagedResultsControl
def gen_fas_searchterms():
alpha = map(chr, range(97, 123))
searchterms = [ alpha_ltr + "*" for alpha_ltr in alpha ]
return searchterms
LDAP_CONF = "/etc/openldap/ldap.conf"
def get_fas_userlist(fas_credentials, search_qry):
creds = fas_credentials
class LDAPClient(object):
fasclient = fedora.client.fas2.AccountSystem(
username=creds['username'],
password=creds['password'],
)
def __init__(self):
self.config = {}
self.conn = None
self._read_config()
timeout = socket.getdefaulttimeout()
socket.setdefaulttimeout(600)
try:
log.info("Downloading FAS cache")
request = fasclient.send_request('/user/list',
req_params={'search': search_qry},
auth=True)
finally:
socket.setdefaulttimeout(timeout)
def _read_config(self):
conf_re = re.compile(r"^([A-Z_]+)\s+(.+)$")
with open(LDAP_CONF) as cf:
for line in cf:
mo = conf_re.match(line.strip())
if mo is None:
continue
variable = mo.group(1)
value = mo.group(2)
self.config[variable] = value
# We don't actually check for CLA+1, just "2 groups"
return [p for p in request['people'] if len(p.memberships) > 1]
def connect(self):
self.conn = ldap.initialize(self.config["URI"].split(" ")[0])
self.conn.protocol_version = 3
self.conn.sasl_interactive_bind_s('', ldap.sasl.gssapi())
def search(self, base, filters, attrs):
page_size = 1000
base_dn = "{base},{main_base}".format(base=base,main_base=self.config['BASE'])
page_cookie = ""
while True:
page_control = SimplePagedResultsControl(
criticality=False, size=page_size, cookie=page_cookie
)
msgid = self.conn.search_ext(
base_dn,
ldap.SCOPE_SUBTREE,
filters,
attrlist=attrs,
serverctrls=[page_control],
)
rtype, rdata, rmsgid, serverctrls = self.conn.result3(msgid)
for dn, obj in rdata:
yield obj
for ctrl in serverctrls:
if isinstance(ctrl, SimplePagedResultsControl):
page_cookie = ctrl.cookie
break
if not page_cookie:
break
def get_fas_userlist(threshold):
os.environ["KRB5_CLIENT_KTNAME"] = fm_config.get("keytab")
ldap_client = LDAPClient()
ldap_client.connect()
filters = "(&(fasCreationTime<={})(objectclass=fasUser))".format(threshold.strftime("%Y%m%d%H%M%SZ"))
response = ldap_client.search(base="cn=users,cn=accounts", filters=filters, attrs=["uid", "memberof"])
for res in response:
groups = []
for groupdn in res.get("memberof", []):
groupdn = groupdn.decode("ascii")
if not groupdn.endswith(",cn=groups,cn=accounts,{}".format(ldap_client.config['BASE'])):
continue
groupname = groupdn.split(",")[0].split("=")[1]
if groupname == "ipausers":
continue # Assume all groups are FAS groups except this one
groups.append(groupname)
yield {"username": res["uid"][0].decode("ascii"), "groups": groups}
def main():
global http_client
now = datetime.datetime.utcnow()
year = datetime.timedelta(days=365.5)
mapping = {
@ -81,26 +124,18 @@ def main():
badge = tahrir.get_badge(badge_id=badge_id)
assert(badge.id)
# Then, do a long query against FAS for our candidates.
# Here I call search terms to generate a lists of search terms
# Looping over the list of search terms, pass the search term to get_fas_userlists
fas_credentials = fm_config['fas_credentials']
searchterms = gen_fas_searchterms()
for search_elem in searchterms:
results = get_fas_userlist(fas_credentials, search_elem)
for badge_id, delta in mapping.items():
badge = tahrir.get_badge(badge_id=badge_id)
for person in results:
creation = datetime.datetime.strptime(
person.creation[:19], '%Y-%m-%d %H:%M:%S')
if now - creation > delta:
hit_em_up(badge, person)
# Then, query IPA for users created before the threshold
for badge_id, delta in mapping.items():
badge = tahrir.get_badge(badge_id=badge_id)
threshold = now - delta
for person in get_fas_userlist(threshold):
if len(person["groups"]) < 2:
continue
hit_em_up(badge, person)
def hit_em_up(badge, fas_user):
email = fas_user.username + "@fedoraproject.org"
email = fas_user["username"] + "@fedoraproject.org"
user = tahrir.get_person(email)
if not user:
@ -110,7 +145,7 @@ def hit_em_up(badge, fas_user):
print email, "already has", badge.id, "skipping."
return
time.sleep(15)
time.sleep(1)
print "awarding", badge.id, "to", email
try:
transaction.begin()

122
roles/badges/backend/files/cron/award-oldschool-badges
View file

@ -6,6 +6,7 @@ __main__.__requires__ = __requires__ = ["tahrir-api", "sqlalchemy>=0.7"];
import pkg_resources
pkg_resources.require(__requires__)
import os
import itertools
import string
import time
@ -14,11 +15,14 @@ import socket
from hashlib import md5
import getpass
import pprint
from collections import defaultdict
from gssapi import Credentials
from gssapi.exceptions import GSSError
from requests_gssapi import HTTPSPNEGOAuth
from tahrir_api.dbapi import TahrirDatabase
import transaction
_fas_cache = {}
import requests
import logging
log = logging.getLogger()
@ -37,88 +41,44 @@ fedmsg.init(**fm_config)
import fedbadges.utils
def user_in_group(user, group_name):
# First, bail out if they're not in the group at all
if not any([g.name == group_name for g in user.memberships]):
return False
# Find the group_id of the group we're looking for..
group_id = None
for g in user.memberships:
if g.name == group_name:
group_id = g.id
break
if not group_id:
return False
# For that group_id, find the relevant role
relevant_role = None
for role in user.roles:
if role.group_id == group_id:
relevant_role = role
break
if not relevant_role:
return False
# They must be actually 'approved' in that group for this to count
return relevant_role.role_status == 'approved'
def get_all_fas_users(creds):
fasclient = fedora.client.fas2.AccountSystem(
username=creds['username'],
password=creds['password'],
)
timeout = socket.getdefaulttimeout()
socket.setdefaulttimeout(600)
def get_http_client():
os.environ["KRB5_CLIENT_KTNAME"] = fm_config.get("keytab")
try:
log.info("Downloading FAS cache")
for a, b in itertools.product(string.lowercase, string.lowercase):
term = a + b + '*'
log.info(" Querying FAS for %r" % term)
request = fasclient.send_request('/user/list',
req_params={'search': term},
auth=True)
log.info(" Found %r matching users." % len(request['people']))
for person in request['people']:
yield person
finally:
socket.setdefaulttimeout(timeout)
creds = Credentials(usage="initiate")
except GSSError as e:
log.error("GSSError trying to authenticate with Kerberos", e)
gssapi_auth = HTTPSPNEGOAuth(opportunistic_auth=True, creds=creds)
session = requests.Session()
session.auth = gssapi_auth
return session
def get_fas_groupings(fas_credentials, lookup, **config):
results = {}
packager_id, ambassadors_id = None, None
sponsor_types = ['sponsor', 'administrator']
mega_list = get_all_fas_users(fas_credentials)
for user in mega_list:
results = defaultdict(list)
membership_types = ("members", "sponsors")
http_client = get_http_client()
for group_name, badge_id in lookup.iteritems():
# This is the main check.
for group_name, badge_id in lookup.iteritems():
if user_in_group(user, group_name):
results[group_name] = results.get(group_name, []) + [user]
for membership_type in membership_types:
url = "%sgroups/%s/%s/" % (
fm_config['fasjson_base_url'],
group_name,
membership_type
)
response = http_client.get(url)
if not response.ok:
continue
for user in response.json()["result"]:
username = user["username"]
results[group_name].append(username)
# Beyond the main check, here is a special check that makes sure they
# are a sponsor in the packager group.
if not packager_id:
for group in user.memberships:
if group.name == 'packager':
packager_id = group.id
if not ambassadors_id:
for group in user.memberships:
if group.name == 'ambassadors':
ambassadors_id = group.id
for role in user.roles:
if role.group_id == packager_id:
if role.role_type in sponsor_types and role.role_status == 'approved':
results['sponsors'] = results.get('sponsors', []) + [user]
if role.group_id == ambassadors_id:
if role.role_type in sponsor_types and role.role_status == 'approved':
results['ambassadors_sponsors'] = results.get('ambassadors_sponsors', []) + [user]
# Beyond the main check, here is a special check for the sponsors of the packager and ambassadors groups
if membership_type != "sponsors":
continue
if group_name == "packager":
results["sponsors"].append(username)
elif group_name == "ambassadors":
results["ambassadors_sponsors"].append(username)
return results
@ -179,9 +139,9 @@ def main():
hit_em_up(badge, members)
def hit_em_up(badge, group):
for fas_user in group:
email = fas_user.username + "@fedoraproject.org"
def hit_em_up(badge, members):
for username in members:
email = username + "@fedoraproject.org"
user = tahrir.get_person(email)
if not user: