From a4cae7e5276647a5267475a7e6ab810988b086ea Mon Sep 17 00:00:00 2001 From: Michael McGrath Date: Mon, 10 Mar 2008 16:32:44 -0500 Subject: [PATCH 01/11] produces identical output to what was there --- fas/fas/templates/group/dump.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fas/fas/templates/group/dump.txt b/fas/fas/templates/group/dump.txt index 6679e3e..565811b 100644 --- a/fas/fas/templates/group/dump.txt +++ b/fas/fas/templates/group/dump.txt @@ -1,3 +1,3 @@ #for role in sorted(group.approved_roles) -${role.member.username},${role.member.emails['primary']},${role.member.human_name},${role.role_type} +${role.member.username},${role.member.emails['primary']},${role.member.human_name},${role.role_type},0 #end From 99ddafc6ad61e4bec71ffb703aea2541f429ed5f Mon Sep 17 00:00:00 2001 From: Michael McGrath Date: Mon, 10 Mar 2008 16:43:03 -0500 Subject: [PATCH 02/11] don't need memberships except for json requests --- fas/fas/group.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/fas/fas/group.py b/fas/fas/group.py index 909a070..e431098 100644 --- a/fas/fas/group.py +++ b/fas/fas/group.py @@ -256,7 +256,9 @@ class Group(controllers.Controller): for group in results: if canViewGroup(person, group): groups.append(group) - memberships[group.name] = group.approved_roles + if self.jsonRequest(): + # Added an efficiency + memberships[group.name] = group.approved_roles if not len(groups): turbogears.flash(_("No Groups found matching '%s'") % search) return dict(groups=groups, search=search, memberships=memberships) From 072f6228768b335df74896e1e904e112acacf6d1 Mon Sep 17 00:00:00 2001 From: Michael McGrath Date: Mon, 10 Mar 2008 18:22:01 -0500 Subject: [PATCH 03/11] What once was an 800 query job is now a 17 query job, used to produce a 1.5M+ file, now produces one around 100K. Hurray efficiency --- fas/client/fasClient | 10 +++++++++- fas/fas/group.py | 20 +++++++++++++++++--- 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/fas/client/fasClient b/fas/client/fasClient index 18f0cff..607a03f 100755 --- a/fas/client/fasClient +++ b/fas/client/fasClient @@ -321,7 +321,15 @@ class MakeShellAccounts(BaseClient): params = {'search' : search} request = self.send_request('group/list', auth=True, input=params) self.groups = request['groups'] - self.memberships = request['memberships'] + memberships = {} + for group in self.groups: + memberships[group['name']] = [] + try: + for member in request['memberships'][u'%s' % group['id']]: + memberships[group['name']].append({'person_id': member}) + except KeyError: + pass + self.memberships = memberships self.valid_groups() return self.groups diff --git a/fas/fas/group.py b/fas/fas/group.py index e431098..48e2474 100644 --- a/fas/fas/group.py +++ b/fas/fas/group.py @@ -3,6 +3,7 @@ from turbogears import controllers, expose, paginate, identity, redirect, widget from turbogears.database import session import cherrypy +import sqlalchemy import fas from fas.auth import * @@ -253,12 +254,25 @@ class Group(controllers.Controller): groups = [] re_search = re.sub(r'\*', r'%', search).lower() results = Groups.query.filter(Groups.name.like(re_search)).order_by('name').all() +# memberships = session.query(PersonRoles).order_by(PersonRoles.sponsor_id).all() + membersql = sqlalchemy.select([PersonRoles.c.person_id, PersonRoles.c.group_id]).order_by(PersonRoles.c.group_id) + members = membersql.execute() + for member in members: + try: + memberships[member[1]].append(member[0]) + except KeyError: + memberships[member[1]]=[member[0]] for group in results: if canViewGroup(person, group): groups.append(group) - if self.jsonRequest(): - # Added an efficiency - memberships[group.name] = group.approved_roles +# if self.jsonRequest(): + # Adds efficiency + #grp_members = session.query(Groups).filter_by(name=group.name).all() +# grp_members = session.query(PersonRoles).filter_by(group_id=group.id).all() +# memberships[group.name] = [] +# for member in grp_members: +# memberships[group.name].append({'person_id': member.person_id}) +# memberships[group.name] = group.approved_roles if not len(groups): turbogears.flash(_("No Groups found matching '%s'") % search) return dict(groups=groups, search=search, memberships=memberships) From c07c1237fe6ac32b1f78d472bccbce9d3844d3d1 Mon Sep 17 00:00:00 2001 From: Michael McGrath Date: Mon, 10 Mar 2008 19:18:58 -0500 Subject: [PATCH 04/11] cleanup --- fas/fas/group.py | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/fas/fas/group.py b/fas/fas/group.py index 48e2474..fe7932f 100644 --- a/fas/fas/group.py +++ b/fas/fas/group.py @@ -254,9 +254,9 @@ class Group(controllers.Controller): groups = [] re_search = re.sub(r'\*', r'%', search).lower() results = Groups.query.filter(Groups.name.like(re_search)).order_by('name').all() -# memberships = session.query(PersonRoles).order_by(PersonRoles.sponsor_id).all() - membersql = sqlalchemy.select([PersonRoles.c.person_id, PersonRoles.c.group_id]).order_by(PersonRoles.c.group_id) - members = membersql.execute() + if self.jsonRequest(): + membersql = sqlalchemy.select([PersonRoles.c.person_id, PersonRoles.c.group_id]).order_by(PersonRoles.c.group_id) + members = membersql.execute() for member in members: try: memberships[member[1]].append(member[0]) @@ -265,14 +265,6 @@ class Group(controllers.Controller): for group in results: if canViewGroup(person, group): groups.append(group) -# if self.jsonRequest(): - # Adds efficiency - #grp_members = session.query(Groups).filter_by(name=group.name).all() -# grp_members = session.query(PersonRoles).filter_by(group_id=group.id).all() -# memberships[group.name] = [] -# for member in grp_members: -# memberships[group.name].append({'person_id': member.person_id}) -# memberships[group.name] = group.approved_roles if not len(groups): turbogears.flash(_("No Groups found matching '%s'") % search) return dict(groups=groups, search=search, memberships=memberships) From 6fd130998f7862a3d3498b3a505c8309aba62fee Mon Sep 17 00:00:00 2001 From: Michael McGrath Date: Mon, 10 Mar 2008 19:43:24 -0500 Subject: [PATCH 05/11] added additional quicknesses to the script. 1) The script now takes about 6 seconds to run against my workstation. 2) User Groups are only created if the host has those users on that OS 3) total IO transfer against a sample migration takes about 2.1M Compression takes that down to about 1.3M. It will likely be worth having our proxy servers do this compression for us. --- fas/client/fasClient | 38 +++++++++++++++++++------------------- fas/fas/user.py | 21 ++++++++++++++++++--- 2 files changed, 37 insertions(+), 22 deletions(-) diff --git a/fas/client/fasClient b/fas/client/fasClient index 607a03f..d059f89 100755 --- a/fas/client/fasClient +++ b/fas/client/fasClient @@ -238,7 +238,7 @@ class MakeShellAccounts(BaseClient): shadow_file = codecs.open(self.temp + '/shadow.txt', mode='w', encoding='utf-8') os.chmod(self.temp + '/shadow.txt', 00400) if not self.people: - self.people = self.people_list() + self.people_list() for person in self.people: username = person['username'] if self.valid_user(username): @@ -281,23 +281,25 @@ class MakeShellAccounts(BaseClient): def groups_text(self, groups=None, people=None): i = 0 file = open(self.temp + '/group.txt', 'w') - if not groups: - groups = self.group_list() - if not people: - people = self.people_list() - - ''' First create all of our users/groups combo ''' - for person in people: - uid = person['id'] - if self.valid_user_group(uid): - username = person['username'] - file.write("=%i %s:x:%i:\n" % (uid, username, uid)) - file.write("0%i %s:x:%i:\n" % (i, username, uid)) - file.write(".%s %s:x:%i:\n" % (username, username, uid)) - i = i + 1 - + if not self.groups: + self.group_list() + if not self.people: + self.people_list() usernames = self.usernames() - for group in groups: + ''' First create all of our users/groups combo ''' + for person in self.people: + uid = person['id'] + try: + if self.valid_user(usernames[uid]): + username = person['username'] + file.write("=%i %s:x:%i:\n" % (uid, username, uid)) + file.write("0%i %s:x:%i:\n" % (i, username, uid)) + file.write(".%s %s:x:%i:\n" % (username, username, uid)) + i = i + 1 + except KeyError: + continue + + for group in self.groups: gid = group['id'] name = group['name'] try: @@ -314,7 +316,6 @@ class MakeShellAccounts(BaseClient): file.write("0%i %s:x:%i:%s\n" % (i, name, gid, memberships)) file.write(".%s %s:x:%i:%s\n" % (name, name, gid, memberships)) i = i + 1 - file.close() def group_list(self, search='*'): @@ -336,7 +337,6 @@ class MakeShellAccounts(BaseClient): def people_list(self, search='*'): params = {'search' : search} self.people = self.send_request('user/list', auth=True, input=params)['people'] - return self.people def email_list(self, search='*'): params = {'search' : search} diff --git a/fas/fas/user.py b/fas/fas/user.py index cf8bb4e..7d0cead 100644 --- a/fas/fas/user.py +++ b/fas/fas/user.py @@ -5,6 +5,8 @@ import cherrypy import turbomail +import sqlalchemy + import os import re import gpgme @@ -280,10 +282,23 @@ class User(controllers.Controller): def list(self, search="a*"): '''List users ''' + re_search = re.sub(r'\*', r'%', search).lower() - people = People.query.filter(People.username.like(re_search)).order_by('username') - if people.count() < 0: - turbogears.flash(_("No users found matching '%s'") % search) + if self.jsonRequest(): + people = [] + peoplesql = sqlalchemy.select([People.c.id, People.c.username, People.c.human_name, People.c.ssh_key, People.c.password]) + persons = peoplesql.execute() + for person in persons: + people.append({ + 'id' : person[0], + 'username' : person[1], + 'human_name' : person[2], + 'ssh_key' : person[3], + 'password' : person[4]}) + else: + people = People.query.filter(People.username.like(re_search)).order_by('username') + if people.count() < 0: + turbogears.flash(_("No users found matching '%s'") % search) return dict(people=people, search=search) @identity.require(turbogears.identity.not_anonymous()) From d6ff6bfffd04f2330fefee2de047c3e53b28eb6b Mon Sep 17 00:00:00 2001 From: Michael McGrath Date: Mon, 10 Mar 2008 19:56:44 -0500 Subject: [PATCH 06/11] getting ready for another release --- fas/fas.spec | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/fas/fas.spec b/fas/fas.spec index 32f7756..970eb24 100644 --- a/fas/fas.spec +++ b/fas/fas.spec @@ -1,7 +1,7 @@ %{!?python_sitelib: %define python_sitelib %(%{__python} -c "from distutils.sysconfig import get_python_lib; print get_python_lib()")} Name: fas -Version: 0.1 +Version: 0.2 Release: 1%{?dist} Summary: Fedora Account System @@ -55,6 +55,10 @@ install -m 0600 fas.cfg $RPM_BUILD_ROOT%{_sysconfdir} %clean rm -rf $RPM_BUILD_ROOT +%pre +/usr/sbin/groupadd -r fas &>/dev/null || : +/usr/sbin/useradd -r -s /sbin/nologin -d /usr/share/fas -M \ + -c 'Fedora Account System User' -g fas fas &>/dev/null || : %files %defattr(-,root,root,-) @@ -67,5 +71,8 @@ rm -rf $RPM_BUILD_ROOT %{_bindir}/* %changelog +* Mon Mar 10 2008 Mike McGrath - 0.2-1 +- Added create user to pre + * Mon Mar 10 2008 Toshio Kuratomi - 0.1-1 - Initial Build. From 1b72d232eef9e55fd033659412073e69f67f3e84 Mon Sep 17 00:00:00 2001 From: Michael McGrath Date: Mon, 10 Mar 2008 19:59:05 -0500 Subject: [PATCH 07/11] making a pretty benign default config --- fas/client/fas.conf | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/fas/client/fas.conf b/fas/client/fas.conf index d878c7a..cf4fbcf 100644 --- a/fas/client/fas.conf +++ b/fas/client/fas.conf @@ -17,16 +17,17 @@ password = admin ; in 'groups' ; groups that should have a shell account on this system. -groups = accounts,fedorabugs +groups = sysadmin-main ; groups that should have a restricted account on this system. ; restricted accounts use the restricted_shell value in [users] -restricted_groups = sysadmin +restricted_groups = ; ssh_restricted_groups: groups that should be restricted by ssh key. You will ; need to disable password based logins in order for this value to have any -; security meaning -ssh_restricted_groups = sysadmin-web +; security meaning. Group types can be placed here as well, for example +; @hg,@git,@svn +ssh_restricted_groups = ; aliases_template: Gets prepended to the aliases file when it is generated by ; fasClient From d05ce0acad256431292980755c9e1e927ab59564 Mon Sep 17 00:00:00 2001 From: Michael McGrath Date: Mon, 10 Mar 2008 20:44:39 -0500 Subject: [PATCH 08/11] updated spec file --- fas/fas.spec | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/fas/fas.spec b/fas/fas.spec index 970eb24..05c1821 100644 --- a/fas/fas.spec +++ b/fas/fas.spec @@ -8,15 +8,16 @@ Summary: Fedora Account System Group: Development/Languages License: GPLv2 URL: https://fedorahosted.org/fas2/ -Source0: https://fedorahosted.org/releases/f/e/fedora-infrastructure/ +Source0: https://fedorahosted.org/releases/f/e/fedora-infrastructure/%{name}-%{version}.tar.gz BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-root-%(%{__id_u} -n) BuildArch: noarch BuildRequires: python-devel -BuildRequires: setuptools-devel +BuildRequires: python-setuptools-devel +BuildRequires: TurboGears Requires: TurboGears >= 1.0.4 Requires: python-sqlalchemy >= 0.4 -Requires: python-turbomail +Requires: python-TurboMail Requires: python-fedora-infrastructure >= 0.2.99.2 %description @@ -33,7 +34,7 @@ Group: Applications/System Requires: python-fedora Requires: rhpl -%description -n clients +%description clients Additional scripts that work as clients to the accounts system. %prep @@ -50,7 +51,7 @@ mkdir -p $RPM_BUILD_ROOT%{_sbindir} mkdir -p $RPM_BUILD_ROOT%{_sysconfdir} mv $RPM_BUILD_ROOT%{_bindir}/start-fas $RPM_BUILD_ROOT%{_sbindir} # Unreadable by others because it's going to contain a database password. -install -m 0600 fas.cfg $RPM_BUILD_ROOT%{_sysconfdir} +install fas.cfg $RPM_BUILD_ROOT%{_sysconfdir} %clean rm -rf $RPM_BUILD_ROOT @@ -58,21 +59,24 @@ rm -rf $RPM_BUILD_ROOT %pre /usr/sbin/groupadd -r fas &>/dev/null || : /usr/sbin/useradd -r -s /sbin/nologin -d /usr/share/fas -M \ - -c 'Fedora Account System User' -g fas fas &>/dev/null || : + -c 'Fedora Acocunt System user' -g fas fas &>/dev/null || : + + %files %defattr(-,root,root,-) %doc README TODO COPYING fas2.sql %{python_sitelib}/* +%{_datadir}/fas/ %{_sbindir}/start-fas -%config(noreplace) %{_sysconfdir}/* +%attr(0640,root,apache) %config(noreplace) %{_sysconfdir}/fas.cfg -%files -n clients +%files clients %{_bindir}/* %changelog * Mon Mar 10 2008 Mike McGrath - 0.2-1 -- Added create user to pre +- Added fas user/group * Mon Mar 10 2008 Toshio Kuratomi - 0.1-1 - Initial Build. From f7fa89b1a37c5b6ba5355183be3aa4921eac634d Mon Sep 17 00:00:00 2001 From: Michael McGrath Date: Mon, 10 Mar 2008 21:20:37 -0500 Subject: [PATCH 09/11] added more requires from the README --- fas/fas.spec | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/fas/fas.spec b/fas/fas.spec index 05c1821..9e177fb 100644 --- a/fas/fas.spec +++ b/fas/fas.spec @@ -19,6 +19,10 @@ Requires: TurboGears >= 1.0.4 Requires: python-sqlalchemy >= 0.4 Requires: python-TurboMail Requires: python-fedora-infrastructure >= 0.2.99.2 +Requires: babel +Requires: pygpgme +Requires: python-babel +Requires: pytz %description The Fedora Account System is a web application that manages the accounts of From a97565bb44a29473febfd66309a6bd75d4ace45d Mon Sep 17 00:00:00 2001 From: Michael McGrath Date: Mon, 10 Mar 2008 22:46:36 -0500 Subject: [PATCH 10/11] This fixes the user's group creations. This script is going to need some attention in the not too distant future. --- fas/client/fasClient | 36 +++++++++++++++++++++++++++++++----- 1 file changed, 31 insertions(+), 5 deletions(-) diff --git a/fas/client/fasClient b/fas/client/fasClient index d059f89..8acfa62 100755 --- a/fas/client/fasClient +++ b/fas/client/fasClient @@ -128,6 +128,7 @@ class MakeShellAccounts(BaseClient): emails = None group_mapping = {} valid_groups = {} + usernames = {} def mk_tempdir(self): self.temp = tempfile.mkdtemp('-tmp', 'fas-', config.get('global', 'temp').strip('"')) @@ -165,6 +166,10 @@ class MakeShellAccounts(BaseClient): def valid_user(self, username): ''' Is the user valid on this system ''' + if not self.valid_groups: + self.valid_groups() + if not self.group_mapping: + self.get_group_mapping() try: for restriction in self.valid_groups: for group in self.valid_groups[restriction]: @@ -267,7 +272,7 @@ class MakeShellAccounts(BaseClient): return True return False - def usernames(self): + def get_usernames(self): usernames = {} if not self.people: self.people_list() @@ -276,7 +281,25 @@ class MakeShellAccounts(BaseClient): if self.valid_user_group(uid): username = person['username'] usernames[uid] = username - return usernames + self.usernames = usernames + + def get_group_mapping(self): + if not self.usernames: + self.get_usernames() + for group in self.groups: + gid = group['id'] + name = group['name'] + try: + ''' Shoot me now I know this isn't right ''' + members = [] + for member in self.memberships[name]: + members.append(self.usernames[member['person_id']]) + memberships = ','.join(members) + self.group_mapping[name] = members + except KeyError: + ''' No users exist in the group ''' + pass + def groups_text(self, groups=None, people=None): i = 0 @@ -285,12 +308,15 @@ class MakeShellAccounts(BaseClient): self.group_list() if not self.people: self.people_list() - usernames = self.usernames() + if not self.usernames: + self.get_usernames() + if not self.group_mapping: + self.get_group_mapping() ''' First create all of our users/groups combo ''' for person in self.people: uid = person['id'] try: - if self.valid_user(usernames[uid]): + if self.valid_user(self.usernames[uid]): username = person['username'] file.write("=%i %s:x:%i:\n" % (uid, username, uid)) file.write("0%i %s:x:%i:\n" % (i, username, uid)) @@ -306,7 +332,7 @@ class MakeShellAccounts(BaseClient): ''' Shoot me now I know this isn't right ''' members = [] for member in self.memberships[name]: - members.append(usernames[member['person_id']]) + members.append(self.usernames[member['person_id']]) memberships = ','.join(members) self.group_mapping[name] = members except KeyError: From f83d85ccc965f420f70fadec20ff070d4a60eb0b Mon Sep 17 00:00:00 2001 From: Michael McGrath Date: Mon, 10 Mar 2008 22:52:51 -0500 Subject: [PATCH 11/11] we don't want to run these if its not a json request --- fas/fas/group.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/fas/fas/group.py b/fas/fas/group.py index fe7932f..553fe73 100644 --- a/fas/fas/group.py +++ b/fas/fas/group.py @@ -257,11 +257,11 @@ class Group(controllers.Controller): if self.jsonRequest(): membersql = sqlalchemy.select([PersonRoles.c.person_id, PersonRoles.c.group_id]).order_by(PersonRoles.c.group_id) members = membersql.execute() - for member in members: - try: - memberships[member[1]].append(member[0]) - except KeyError: - memberships[member[1]]=[member[0]] + for member in members: + try: + memberships[member[1]].append(member[0]) + except KeyError: + memberships[member[1]]=[member[0]] for group in results: if canViewGroup(person, group): groups.append(group)