Running Standalone SQLAlchemy Scripts in Pyramid

From time to time there comes the need to run automated scripts managed by either Python or Cron. In a recent project I had to run a standalone script that checks records on a database at certain hours. So here’s what I came up with. Most of the script is based on this post by Daniel Mayer.

from paste.deploy import appconfig
 
from sqlalchemy import engine_from_config
 
from sqlalchemy.orm.exc import NoResultFound, MultipleResultsFound
 
#here is where sqlalchemy objects are imported
from PRJ.models import DBSession as db,DBRecords
 
#import the session manager. This way commits will be handled automatically by the Zope Transaction Manager
import transaction
 
# Load Application Configuration and Return as Dictionary
conf = appconfig('config:' + 'development.ini', 
        relative_to="/var/www/PRJ",
        name="main")
 
# Bind Engine Based on Config
engine = engine_from_config(conf, 'sqlalchemy.')
db.configure(bind=engine)
 
#Query the DB
data = db.query(DBRecords).one()
 
with transaction.manager:
  for record in data:
    #query or update DB
 

Host Email with Postfix, Dovecot and Postgres on Debian 6 (Squeeze)

Recently I had to configure a web server from scratch, which I haven’t done in a while. So I took a look at the guides from my current hosting provider Linode. They have a guide covering debian 5 (Lenny) which is basically the same for debian 6 (squeeze). To make this a bit more exciting I decided to replace MySQL for Postgres since I really didn’t need it. So I will only post about the changes you need to perform based from the Linode guide.

Install Required Packages

Replace the mysql related packages for postgres and replace mailx for bsd-mailx.

apt-get install postfix postfix-pgsql postfix-doc dovecot-common dovecot-imapd dovecot-pop3d postfix-tls libsasl2-2 libsasl2-modules libsasl2-modules-sql sasl2-bin libpam-pgsql openssl telnet bsd-mailx postgresql-contrib postgresql-8.4

Set up Postgres for Virtual Domains and Users

Open /etc/postgres/8.4/main/pg_hba.conf and make sure local connections require a password

host    all         all         127.0.0.1/32          md5

You could also make all local connections trusted and in this case no password will be needed, but of course it could represent a security risk

host    all         all         127.0.0.1/32          trust

This next step is optional but as a security measure I like to have Postgres listen on a diferent port other than the default.

Open /etc/postgres/8.4/main/postgres.conf and uncomment the following line and add 6789 as the port number

port = 6789

Create the user for the mail database.

sudo -u postgres createuser  -P -E -s -d mail_admin

Now create the database.

sudo -u postgres createdb -O mail_admin mail -E UTF-8

Grant all privileges.

GRANT ALL PRIVILEGES ON DATABASE mail to mail_admin;

Create domains, forwardings, users and transport tables.

CREATE TABLE domains (domain varchar(50) NOT NULL, PRIMARY KEY (domain) );
CREATE TABLE forwardings (source varchar(80) NOT NULL, destination TEXT NOT NULL, PRIMARY KEY (source) );
CREATE TABLE users (email varchar(80) NOT NULL, password varchar(20) NOT NULL, PRIMARY KEY (email) );
CREATE TABLE transport ( domain varchar(128) NOT NULL default '', transport varchar(128) NOT NULL default '', constraint fieldomainunique UNIQUE  (domain) );

Now I’ll create a domain and user in mail database. But first I will use Postgres pgcrypto functions since passwords need to be encrypted using the UNIX original encryption scheme. Replace your_password with your own password.

mail=#\i /usr/share/postgresql/8.4/contrib/pgcrypto.sql
INSERT INTO domains (domain) VALUES ('example.com');
INSERT INTO users (email, password) VALUES ('info@example.com', crypt('your_password', gen_salt('des') ));
mail=#\q

Configure Postfix to work with Postgres

Create file /etc/postfix/pgsql-virtual_domains.cf

user = mail_admin
password = mail_admin_password
dbname = mail
query = SELECT domain AS virtual FROM domains WHERE domain='%s'
hosts = 127.0.0.1:6789

Create file /etc/postfix/pgsql-virtual_forwardings.cf

user = mail_admin
password = mail_admin_password
dbname = mail
query = SELECT destination FROM forwardings WHERE source='%s'
hosts = 127.0.0.1:6789

Create file /etc/postfix/pgsql-virtual_mailboxes.cf

user = mail_admin
password = mail_admin_password
dbname = mail
query = SELECT split_part(email, '@', 2) || '/' || split_part(email,'@', 1) FROM users WHERE email='%s';
hosts = 127.0.0.1:6789

Create file /etc/postfix/pgsql-virtual_email2email.cf

user = mail_admin
password = mail_admin_password
dbname = mail
query = SELECT email FROM users WHERE email='%s'
hosts = 127.0.0.1:6789

Set the proper permissions

chmod o= /etc/postfix/pgsql-virtual_*.cf
chgrp postfix /etc/postfix/pgsql-virtual_*.cf

Add the following one line at a time. Replace qualified.hostname.com with your hostname

postconf -e 'myhostname = qualified.hostname.com'
postconf -e 'mydestination = qualified.hostname.com, localhost, localhost.localdomain'
postconf -e 'mynetworks = 127.0.0.0/8'
postconf -e 'message_size_limit = 30720000'
postconf -e 'virtual_alias_domains ='
postconf -e 'virtual_alias_maps = proxy:pgsql:/etc/postfix/pgsql-virtual_forwardings.cf, pgsql:/etc/postfix/pgsql-virtual_email2email.cf'
postconf -e 'virtual_mailbox_domains = proxy:pgsql:/etc/postfix/pgsql-virtual_domains.cf'
postconf -e 'virtual_mailbox_maps = proxy:pgsql:/etc/postfix/pgsql-virtual_mailboxes.cf'
postconf -e 'virtual_mailbox_base = /home/vmail'postconf -e 'virtual_uid_maps = static:5000'postconf -e 'virtual_gid_maps = static:5000'
postconf -e 'smtpd_sasl_auth_enable = yes'postconf -e 'broken_sasl_auth_clients = yes'
postconf -e 'smtpd_sasl_authenticated_header = yes'
postconf -e 'smtpd_recipient_restrictions = permit_mynetworks, permit_sasl_authenticated, reject_unauth_destination'
postconf -e 'smtpd_use_tls = yes'
postconf -e 'smtpd_tls_cert_file = /etc/postfix/smtpd.cert'
postconf -e 'smtpd_tls_key_file = /etc/postfix/smtpd.key'
postconf -e 'virtual_create_maildirsize = yes'
postconf -e 'virtual_maildir_extended = yes'
postconf -e 'proxy_read_maps = $local_recipient_maps $mydestination $virtual_alias_maps $virtual_alias_domains $virtual_mailbox_maps $virtual_mailbox_domains $relay_recipient_maps $relay_domains $canonical_maps $sender_canonical_maps $recipient_canonical_maps $relocated_maps $transport_maps $mynetworks $virtual_mailbox_limit_maps'
postconf -e virtual_transport=dovecotpostconf -e dovecot_destination_recipient_limit=1

Change postfix listening port to 2525 in /etc/postfix/master.cf.

2525      inet  n       -       -       -       -       smtpd

Or you could just uncomment the submission line and default to port 587.

submission inet n       -       -       -       -       smtpd

Configure saslauthd to use Postgres

Create file /etc/pam.d/smtp. Replace your_password with your own password.

auth    required   pam_pgsql.so user=mail_admin passwd=your_password host=127.0.0.1 port=6789 db=mail table=users usercolumn=email passwdcolumn=password crypt=1
account sufficient pam_pgsql.so user=mail_admin passwd=your_password host=127.0.0.1 port=6789 db=mail table=users usercolumn=email passwdcolumn=password crypt=1

Create file /etc/postfix/sasl/smtpd.conf. Replace your_password with your own password.

pwcheck_method: saslauthd
mech_list: plain login
allow_plaintext: true
auxprop_plugin: sql
sql_engine: pgsql
sql_hostnames: 127.0.0.1:6789
sql_user: mail_admin
sql_passwd: your_password
sql_database: mail
sql_select: select password from users where email = '%u'

Create file /etc/pam_pgsql.conf. Replace your_password with your own password.

database = mail
host = localhost
user = mail_admin
password = your_password
table = users
user_column = email
pwd_column = password
pw_type = crypt

Set proper permissions to files.

chmod o= /etc/pam_pgsql.conf
chmod o= /etc/postfix/sasl/smtpd.conf

Configure Dovecot

One thing that was pointed out at the end of the guide by user Nathan  is to change cmusieve for sieve on file /etc/dovecot/dovecot.conf on this following part

protocol lda {
    log_path = /home/vmail/dovecot-deliver.log
    auth_socket_path = /var/run/dovecot/auth-master
    postmaster_address = postmaster@example.com
    mail_plugins = cmusieve #change to sieve
    global_script_path = /home/vmail/globalsieverc
}

Create file /etc/dovecot/dovecot-sql.conf. Replace your_password with your own password.

driver = pgsql
connect = host=127.0.0.1 port=6789 dbname=mail user=mail_admin password=your_password
default_pass_scheme = CRYPT
password_query = SELECT email as user, password FROM users WHERE email='%u';


And thats about it. If you managed to configure everything correctly congratulations!, if not post a comment and we’ll try to find the solution.

Troubleshooting

Remember that /var/log/syslog and /var/log/mail.log are your friends. You can also add the word debug at the end of each line on /etc/pam.d/smtp to help you troubleshoot any problems you may encounter.

 

repoze.what with reflected tables using SQLAlchemy in Pylons

One of the many things I like about SQLAlchemy is the feature of reflected tables, this means I can develop dashboards for existing applications with my current favorite framework(Pylons). So I had to create an authorization mechanism for my dashboard and this recipe from the Pylons cookbook was just what I needed.

The only thing I had to change was the way my tables were being mapped since the users table schema was already defined. This is what I did:

user_table = schema.Table('users', Base.metadata, autoload=True, autoload_with=engine)
 
user_group_table = schema.Table('user_groups',Base.metadata,autoload=True,autoload_with=engine)
 
group_permission_table = schema.Table('group_permission',Base.metadata,autoload=True,autoload_with=engine)
 
permission_table = schema.Table('permissions', Base.metadata,autoload=True,autoload_with=engine)
 
group_table = schema.Table('groups', Base.metadata,autoload=True,autoload_with=engine)
 
orm.mapper(User,user_table, properties={
      'groups':orm.relation(Group, secondary=user_group_table),
    })
 
orm.mapper(Permission,permission_table,properties={
      'groups':orm.relation(Group, secondary=group_permission_table),
    })
 
orm.mapper(Group,group_table,properties={
      'permissions':orm.relation(Permission, secondary=group_permission_table),
      'users':orm.relation(User, secondary=user_group_table),
    })
 
orm.mapper(UGT,user_group_table)
 
orm.mapper(GPT,group_permission_table)

And here’s the schema I had to create, except for the users table:

CREATE TABLE user_groups( 
id serial NOT NULL,  
user_id INTEGER,  
group_id INTEGER,  
CONSTRAINT user_groups_pkey PRIMARY KEY (id), 
CONSTRAINT gid_fkey FOREIGN KEY (group_id) REFERENCES groups (id)
CONSTRAINT uid_fkey FOREIGN KEY (user_id)   REFERENCES users (id) 
)
CREATE TABLE group_permission(
id serial NOT NULL,
gid INTEGER,
permid INTEGER,
CONSTRAINT group_permission_pkey PRIMARY KEY (id),
CONSTRAINT groupid_fkey FOREIGN KEY (gid) REFERENCES groups (id),
CONSTRAINT permid_fkey FOREIGN KEY (permid) REFERENCES permissions (id) 
)
CREATE TABLE permissions(
  id serial NOT NULL,
  pname CHARACTER VARYING(50),
  CONSTRAINT permissions_pkey PRIMARY KEY (id)
)
CREATE TABLE groups(
  id serial NOT NULL,
  gname CHARACTER VARYING(50),
  CONSTRAINT groups_pkey PRIMARY KEY (id)
)
CREATE TABLE users(
id serial NOT NULL,
username CHARACTER VARYING(50),
email CHARACTER VARYING(50),
upassword CHARACTER VARYING(150),
CONSTRAINT users_pkey PRIMARY KEY (id)
)

And that’s all there’s to it. I was very surprise on how flexible SQLAlchemy is.