diff --git a/roles/badges/backend/files/cron/award-badges-dev-badge b/roles/badges/backend/files/cron/award-badges-dev-badge index f161718a58..1c812c789a 100644 --- a/roles/badges/backend/files/cron/award-badges-dev-badge +++ b/roles/badges/backend/files/cron/award-badges-dev-badge @@ -12,9 +12,13 @@ import os import subprocess import shutil +from gssapi import Credentials +from gssapi.exceptions import GSSError +from requests_gssapi import HTTPSPNEGOAuth from tahrir_api.dbapi import TahrirDatabase import fedbadges.utils import transaction +import requests import fedora.client import socket @@ -34,33 +38,19 @@ fedmsg.init(**fm_config) badge = None -email_to_username_mapping = {} - -_fas_cache = {} +http_client = None -def make_email_to_username_mapping(fas_credentials): - fasclient = fedora.client.fas2.AccountSystem( - username=fas_credentials['username'], - password=fas_credentials['password'], - ) - - timeout = socket.getdefaulttimeout() - socket.setdefaulttimeout(600) +def get_http_client(): + os.environ["KRB5_CLIENT_KTNAME"] = fm_config.get("keytab") try: - log.info("Downloading FAS cache") - request = fasclient.send_request('/user/list', - req_params={'search': '*'}, - auth=True) - finally: - socket.setdefaulttimeout(timeout) - - results = {} - for person in request['people']: - if person.email: - results[person.email] = person.username - - return results + creds = Credentials(usage="initiate") + except GSSError as e: + log.error("GSSError trying to authenticate with Kerberos", e) + gssapi_auth = HTTPSPNEGOAuth(opportunistic_auth=True, creds=creds) + session = requests.Session() + session.auth = gssapi_auth + return session def clone_repo(repo): @@ -88,26 +78,20 @@ def gather_authors(location): os.chdir(pwd) return authors + def emails_to_fas_accounts(emails): usernames = [] for email in emails: - if email.endswith('@fedoraproject.org'): - username = email[:-1 * len('@fedoraproject.org')] - if not username in usernames: - usernames.append(username) + result = http_client.get( + "{}search/users/".format(fm_config['fasjson_base_url']), + params={"email": email} + ) + if not result.ok: continue - - # Otherwise, look it up in FAS and append - raise WTFError("wtf is going on with fas") - - return usernames - - -def emails_to_fas_accounts(emails): - usernames = [] - for email in emails: - if email in email_to_username_mapping: - usernames.append(email_to_username_mapping[email]) + response = result.json() + if response["page"]["total_results"] != 1: + continue + usernames.append(response["result"][0]["username"]) return usernames @@ -153,6 +137,5 @@ if __name__ == '__main__': badge = tahrir.get_badge(badge_id='badge-off!') if not badge: raise ValueError("badge does not exist") - email_to_username_mapping = make_email_to_username_mapping( - fm_config['fas_credentials']) + http_client = get_http_client() main() diff --git a/roles/badges/backend/files/cron/award-flock-paparazzi-badge b/roles/badges/backend/files/cron/award-flock-paparazzi-badge index 42b19b3d7d..b35c9d9b85 100644 --- a/roles/badges/backend/files/cron/award-flock-paparazzi-badge +++ b/roles/badges/backend/files/cron/award-flock-paparazzi-badge @@ -11,19 +11,20 @@ Author: Ralph Bean from __future__ import print_function import __main__ -__main__.__requires__ = __requires__ = ["tahrir-api", "sqlalchemy>=0.7"]; +__main__.__requires__ = __requires__ = ["tahrir-api", "sqlalchemy>=0.7"] import pkg_resources pkg_resources.require(__requires__) -import socket import time -import getpass +import os import ConfigParser import requests -import fedora.client import transaction import tahrir_api.dbapi +from gssapi import Credentials +from gssapi.exceptions import GSSError +from requests_gssapi import HTTPSPNEGOAuth import fedmsg import fedmsg.config @@ -43,8 +44,6 @@ config.read(['flock-paparazzi.ini', '/etc/flock-paparazzi.ini']) flickr_api_key = config.get('general', 'flickr_api_key') g_plus_key = config.get('general', 'g_plus_key') userIP = config.get('general', 'userIP') -fas_username = config.get('general', 'fas_username') -fas_password = config.get('general', 'fas_password') # API urls flickr_url = 'https://api.flickr.com/services/rest/' @@ -52,7 +51,19 @@ g_plus_url = 'https://www.googleapis.com/plus/v1/activities' badge_id = 'flock-paparazzi' -_fas_cache = {} +http_client = None + + +def get_http_client(): + os.environ["KRB5_CLIENT_KTNAME"] = fm_config.get("keytab") + try: + creds = Credentials(usage="initiate") + except GSSError as e: + print("GSSError trying to authenticate with Kerberos", e) + gssapi_auth = HTTPSPNEGOAuth(opportunistic_auth=True, creds=creds) + session = requests.Session() + session.auth = gssapi_auth + return session def get_g_plus_persons(query): @@ -148,50 +159,24 @@ def get_flickr_persons(tags): yield value -def make_fas_cache(username, password): - global _fas_cache - if _fas_cache: - return _fas_cache - - print("No previous fas cache found. Looking to rebuild.") - - try: - import fedora.client.fas2 - except ImportError: - print("No python-fedora installed. Not caching fas.") - return {} - - if not username or not password: - print("No fas credentials found. Not caching fas.") - return {} - - fasclient = fedora.client.fas2.AccountSystem( - username=username, - password=password, +def get_username(name): + # First, check if the same username exists: + response = http_client.get( + "%susers/%s/" % (fm_config['fasjson_base_url'], name) ) - - timeout = socket.getdefaulttimeout() - socket.setdefaulttimeout(600) - try: - print("Downloading FAS cache...") - request = fasclient.send_request( - '/user/list', - req_params={'search': '*'}, - auth=True, - ) - finally: - socket.setdefaulttimeout(timeout) - - print("Caching necessary user data") - for user in request['people']: - for key in ['username', 'human_name']: - if user[key]: - _fas_cache[user[key]] = user['username'] - - del request - del fasclient - - return _fas_cache + if response.ok: + return name + # Now try with the human name + response = http_client.get( + "%ssearch/users/" % fm_config['fasjson_base_url'], + params={"human_name": name} + ) + if not response.ok: + return None + response = response.json() + if response["page"]["total_results"] != 1: + return None + return response["result"][0]["username"] def get_persons(): @@ -206,6 +191,7 @@ def get_persons(): def main(): + global http_client # First, initialize the tahrir db connection uri = fm_config['badges_global']['database_uri'] tahrir = tahrir_api.dbapi.TahrirDatabase( @@ -213,8 +199,7 @@ def main(): notification_callback=fedbadges.utils.notification_callback, ) - # Then, build a fas cache. this takes forever.. - cache = make_fas_cache(fas_username, fas_password) + http_client = get_http_client() badge = tahrir.get_badge(badge_id) already_has_it = [a.person.nickname for a in badge.assertions] @@ -231,14 +216,15 @@ def main(): continue print("* Considering", person) - if person in cache: - if cache[person] in already_has_it: - print("Skipping %r" % cache[person]) + username = get_username(person) + if username is not None: + if username in already_has_it: + print("Skipping %r" % username) continue - print(" *", cache[person], "gets the badge") - already_has_it.append(cache[person]) - email = cache[person] + "@fedoraproject.org" + print(" *", username, "gets the badge") + already_has_it.append(username) + email = username + "@fedoraproject.org" try: transaction.begin() tahrir.add_assertion(badge_id, email, None) diff --git a/roles/badges/backend/files/cron/award-lifecycle-badges b/roles/badges/backend/files/cron/award-lifecycle-badges index 1982b29f08..d8c39e9815 100755 --- a/roles/badges/backend/files/cron/award-lifecycle-badges +++ b/roles/badges/backend/files/cron/award-lifecycle-badges @@ -6,20 +6,19 @@ __main__.__requires__ = __requires__ = ["tahrir-api", "sqlalchemy>=0.7"]; import pkg_resources pkg_resources.require(__requires__) +import os import datetime import time import urllib import socket +import logging +import re from tahrir_api.dbapi import TahrirDatabase import transaction -_fas_cache = {} - -import logging log = logging.getLogger() logging.basicConfig() -import fedora.client.fas2 import fedmsg import fedmsg.config @@ -32,39 +31,83 @@ fedmsg.init(**fm_config) import fedbadges.utils -# generates a list of search terms -# alpha map is just a lowercase english alphabet +import ldap +import ldap.sasl +from ldap.controls.libldap import SimplePagedResultsControl -def gen_fas_searchterms(): - alpha = map(chr, range(97, 123)) - searchterms = [ alpha_ltr + "*" for alpha_ltr in alpha ] - return searchterms +LDAP_CONF = "/etc/openldap/ldap.conf" -def get_fas_userlist(fas_credentials, search_qry): - creds = fas_credentials +class LDAPClient(object): - fasclient = fedora.client.fas2.AccountSystem( - username=creds['username'], - password=creds['password'], - ) + def __init__(self): + self.config = {} + self.conn = None + self._read_config() - timeout = socket.getdefaulttimeout() - socket.setdefaulttimeout(600) - try: - log.info("Downloading FAS cache") - request = fasclient.send_request('/user/list', - req_params={'search': search_qry}, - auth=True) - - finally: - socket.setdefaulttimeout(timeout) + def _read_config(self): + conf_re = re.compile(r"^([A-Z_]+)\s+(.+)$") + with open(LDAP_CONF) as cf: + for line in cf: + mo = conf_re.match(line.strip()) + if mo is None: + continue + variable = mo.group(1) + value = mo.group(2) + self.config[variable] = value - # We don't actually check for CLA+1, just "2 groups" - return [p for p in request['people'] if len(p.memberships) > 1] + def connect(self): + self.conn = ldap.initialize(self.config["URI"].split(" ")[0]) + self.conn.protocol_version = 3 + self.conn.sasl_interactive_bind_s('', ldap.sasl.gssapi()) + + def search(self, base, filters, attrs): + page_size = 1000 + base_dn = "{base},{main_base}".format(base=base,main_base=self.config['BASE']) + page_cookie = "" + while True: + page_control = SimplePagedResultsControl( + criticality=False, size=page_size, cookie=page_cookie + ) + msgid = self.conn.search_ext( + base_dn, + ldap.SCOPE_SUBTREE, + filters, + attrlist=attrs, + serverctrls=[page_control], + ) + rtype, rdata, rmsgid, serverctrls = self.conn.result3(msgid) + for dn, obj in rdata: + yield obj + for ctrl in serverctrls: + if isinstance(ctrl, SimplePagedResultsControl): + page_cookie = ctrl.cookie + break + if not page_cookie: + break + + +def get_fas_userlist(threshold): + os.environ["KRB5_CLIENT_KTNAME"] = fm_config.get("keytab") + ldap_client = LDAPClient() + ldap_client.connect() + filters = "(&(fasCreationTime<={})(objectclass=fasUser))".format(threshold.strftime("%Y%m%d%H%M%SZ")) + response = ldap_client.search(base="cn=users,cn=accounts", filters=filters, attrs=["uid", "memberof"]) + for res in response: + groups = [] + for groupdn in res.get("memberof", []): + groupdn = groupdn.decode("ascii") + if not groupdn.endswith(",cn=groups,cn=accounts,{}".format(ldap_client.config['BASE'])): + continue + groupname = groupdn.split(",")[0].split("=")[1] + if groupname == "ipausers": + continue # Assume all groups are FAS groups except this one + groups.append(groupname) + yield {"username": res["uid"][0].decode("ascii"), "groups": groups} def main(): + global http_client now = datetime.datetime.utcnow() year = datetime.timedelta(days=365.5) mapping = { @@ -81,26 +124,18 @@ def main(): badge = tahrir.get_badge(badge_id=badge_id) assert(badge.id) - # Then, do a long query against FAS for our candidates. - # Here I call search terms to generate a lists of search terms - # Looping over the list of search terms, pass the search term to get_fas_userlists - - fas_credentials = fm_config['fas_credentials'] - searchterms = gen_fas_searchterms() - for search_elem in searchterms: - results = get_fas_userlist(fas_credentials, search_elem) - - for badge_id, delta in mapping.items(): - badge = tahrir.get_badge(badge_id=badge_id) - for person in results: - creation = datetime.datetime.strptime( - person.creation[:19], '%Y-%m-%d %H:%M:%S') - if now - creation > delta: - hit_em_up(badge, person) + # Then, query IPA for users created before the threshold + for badge_id, delta in mapping.items(): + badge = tahrir.get_badge(badge_id=badge_id) + threshold = now - delta + for person in get_fas_userlist(threshold): + if len(person["groups"]) < 2: + continue + hit_em_up(badge, person) def hit_em_up(badge, fas_user): - email = fas_user.username + "@fedoraproject.org" + email = fas_user["username"] + "@fedoraproject.org" user = tahrir.get_person(email) if not user: @@ -110,7 +145,7 @@ def hit_em_up(badge, fas_user): print email, "already has", badge.id, "skipping." return - time.sleep(15) + time.sleep(1) print "awarding", badge.id, "to", email try: transaction.begin() diff --git a/roles/badges/backend/files/cron/award-oldschool-badges b/roles/badges/backend/files/cron/award-oldschool-badges index 2fed195f14..234b5df3ea 100644 --- a/roles/badges/backend/files/cron/award-oldschool-badges +++ b/roles/badges/backend/files/cron/award-oldschool-badges @@ -6,6 +6,7 @@ __main__.__requires__ = __requires__ = ["tahrir-api", "sqlalchemy>=0.7"]; import pkg_resources pkg_resources.require(__requires__) +import os import itertools import string import time @@ -14,11 +15,14 @@ import socket from hashlib import md5 import getpass import pprint +from collections import defaultdict +from gssapi import Credentials +from gssapi.exceptions import GSSError +from requests_gssapi import HTTPSPNEGOAuth from tahrir_api.dbapi import TahrirDatabase import transaction - -_fas_cache = {} +import requests import logging log = logging.getLogger() @@ -37,88 +41,44 @@ fedmsg.init(**fm_config) import fedbadges.utils -def user_in_group(user, group_name): - # First, bail out if they're not in the group at all - if not any([g.name == group_name for g in user.memberships]): - return False - - # Find the group_id of the group we're looking for.. - group_id = None - for g in user.memberships: - if g.name == group_name: - group_id = g.id - break - - if not group_id: - return False - - # For that group_id, find the relevant role - relevant_role = None - for role in user.roles: - if role.group_id == group_id: - relevant_role = role - break - - if not relevant_role: - return False - - # They must be actually 'approved' in that group for this to count - return relevant_role.role_status == 'approved' - -def get_all_fas_users(creds): - fasclient = fedora.client.fas2.AccountSystem( - username=creds['username'], - password=creds['password'], - ) - - timeout = socket.getdefaulttimeout() - socket.setdefaulttimeout(600) +def get_http_client(): + os.environ["KRB5_CLIENT_KTNAME"] = fm_config.get("keytab") try: - log.info("Downloading FAS cache") - for a, b in itertools.product(string.lowercase, string.lowercase): - term = a + b + '*' - log.info(" Querying FAS for %r" % term) - request = fasclient.send_request('/user/list', - req_params={'search': term}, - auth=True) - log.info(" Found %r matching users." % len(request['people'])) - for person in request['people']: - yield person - finally: - socket.setdefaulttimeout(timeout) + creds = Credentials(usage="initiate") + except GSSError as e: + log.error("GSSError trying to authenticate with Kerberos", e) + gssapi_auth = HTTPSPNEGOAuth(opportunistic_auth=True, creds=creds) + session = requests.Session() + session.auth = gssapi_auth + return session def get_fas_groupings(fas_credentials, lookup, **config): - results = {} - packager_id, ambassadors_id = None, None - sponsor_types = ['sponsor', 'administrator'] - mega_list = get_all_fas_users(fas_credentials) - for user in mega_list: + results = defaultdict(list) + membership_types = ("members", "sponsors") + http_client = get_http_client() + for group_name, badge_id in lookup.iteritems(): # This is the main check. - for group_name, badge_id in lookup.iteritems(): - if user_in_group(user, group_name): - results[group_name] = results.get(group_name, []) + [user] + for membership_type in membership_types: + url = "%sgroups/%s/%s/" % ( + fm_config['fasjson_base_url'], + group_name, + membership_type + ) + response = http_client.get(url) + if not response.ok: + continue + for user in response.json()["result"]: + username = user["username"] + results[group_name].append(username) - # Beyond the main check, here is a special check that makes sure they - # are a sponsor in the packager group. - if not packager_id: - for group in user.memberships: - if group.name == 'packager': - packager_id = group.id - - if not ambassadors_id: - for group in user.memberships: - if group.name == 'ambassadors': - ambassadors_id = group.id - - for role in user.roles: - if role.group_id == packager_id: - if role.role_type in sponsor_types and role.role_status == 'approved': - results['sponsors'] = results.get('sponsors', []) + [user] - - if role.group_id == ambassadors_id: - if role.role_type in sponsor_types and role.role_status == 'approved': - results['ambassadors_sponsors'] = results.get('ambassadors_sponsors', []) + [user] + # Beyond the main check, here is a special check for the sponsors of the packager and ambassadors groups + if membership_type != "sponsors": + continue + if group_name == "packager": + results["sponsors"].append(username) + elif group_name == "ambassadors": + results["ambassadors_sponsors"].append(username) return results @@ -179,9 +139,9 @@ def main(): hit_em_up(badge, members) -def hit_em_up(badge, group): - for fas_user in group: - email = fas_user.username + "@fedoraproject.org" +def hit_em_up(badge, members): + for username in members: + email = username + "@fedoraproject.org" user = tahrir.get_person(email) if not user: