DB

by webjunkie on Cron, DB, Paste, Pyramid, Python, SQLAlchemy

Running Standalone SQLAlchemy Scripts in Pyramid

From time to time there comes the need to run automated scripts managed by either Python or Cron. In a recent project I had to run a standalone script that checks records on a database at certain hours. So here’s what I came up with. Most of the script is based on this post by Daniel Mayer.

from paste.deploy import appconfig
from sqlalchemy import engine_from_config
from sqlalchemy.orm.exc import NoResultFound, MultipleResultsFound
#here is where sqlalchemy objects are imported
from PRJ.models import DBSession as db,DBRecords
#import the session manager. This way commits will be handled automatically by the Zope Transaction Manager
import transaction
# Load Application Configuration and Return as Dictionary
conf = appconfig('config:' + 'development.ini',
        relative_to="/var/www/PRJ",
        name="main")
# Bind Engine Based on Config
engine = engine_from_config(conf, 'sqlalchemy.')
db.configure(bind=engine)
#Query the DB
data = db.query(DBRecords).one()
with transaction.manager:
  for record in data:
    #query or update DB
 
by webjunkie on DB, Debian, Dovecot, Postfix, Postgres, uWSGI

Host Email with Postfix, Dovecot and Postgres on Debian 6 (Squeeze)

Recently I had to configure a web server from scratch, which I haven’t done in a while. So I took a look at the guides from my current hosting provider Linode. They have a guide covering debian 5 (Lenny) which is basically the same for debian 6 (squeeze). To make this a bit more exciting I decided to replace MySQL for Postgres since I really didn’t need it. So I will only post about the changes you need to perform based from the Linode guide.

Install Required Packages

Replace the mysql related packages for postgres and replace mailx for bsd-mailx.

apt-get install postfix postfix-pgsql postfix-doc dovecot-common dovecot-imapd dovecot-pop3d postfix-tls libsasl2-2 libsasl2-modules libsasl2-modules-sql sasl2-bin libpam-pgsql openssl telnet bsd-mailx postgresql-contrib postgresql-8.4

Set up Postgres for Virtual Domains and Users

Open /etc/postgres/8.4/main/pg_hba.conf and make sure local connections require a password

host    all         all         127.0.0.1/32          md5

You could also make all local connections trusted and in this case no password will be needed, but of course it could represent a security risk

host    all         all         127.0.0.1/32          trust

This next step is optional but as a security measure I like to have Postgres listen on a diferent port other than the default.
Open /etc/postgres/8.4/main/postgres.conf and uncomment the following line and add 6789 as the port number

port = 6789

Create the user for the mail database.

sudo -u postgres createuser  -P -E -s -d mail_admin

Now create the database.

sudo -u postgres createdb -O mail_admin mail -E UTF-8

Grant all privileges.

GRANT ALL PRIVILEGES ON DATABASE mail to mail_admin;

Create domains, forwardings, users and transport tables.

CREATE TABLE domains (domain varchar(50) NOT NULL, PRIMARY KEY (domain) );
CREATE TABLE forwardings (source varchar(80) NOT NULL, destination TEXT NOT NULL, PRIMARY KEY (source) );
CREATE TABLE users (email varchar(80) NOT NULL, password varchar(20) NOT NULL, PRIMARY KEY (email) );
CREATE TABLE transport ( domain varchar(128) NOT NULL default '', transport varchar(128) NOT NULL default '', constraint fieldomainunique UNIQUE  (domain) );

Now I’ll create a domain and user in mail database. But first I will use Postgres pgcrypto functions since passwords need to be encrypted using the UNIX original encryption scheme. Replace your_password with your own password.

mail=#\i /usr/share/postgresql/8.4/contrib/pgcrypto.sql
INSERT INTO domains (domain) VALUES ('example.com');
INSERT INTO users (email, password) VALUES ('info@example.com', crypt('your_password', gen_salt('des') ));
mail=#\q

Configure Postfix to work with Postgres

Create file /etc/postfix/pgsql-virtual_domains.cf

user = mail_admin
password = mail_admin_password
dbname = mail
query = SELECT domain AS virtual FROM domains WHERE domain='%s'
hosts = 127.0.0.1:6789

Create file /etc/postfix/pgsql-virtual_forwardings.cf

user = mail_admin
password = mail_admin_password
dbname = mail
query = SELECT destination FROM forwardings WHERE source='%s'
hosts = 127.0.0.1:6789

Create file /etc/postfix/pgsql-virtual_mailboxes.cf

user = mail_admin
password = mail_admin_password
dbname = mail
query = SELECT split_part(email, '@', 2) || '/' || split_part(email,'@', 1) FROM users WHERE email='%s';
hosts = 127.0.0.1:6789

Create file /etc/postfix/pgsql-virtual_email2email.cf

user = mail_admin
password = mail_admin_password
dbname = mail
query = SELECT email FROM users WHERE email='%s'
hosts = 127.0.0.1:6789

Set the proper permissions

chmod o= /etc/postfix/pgsql-virtual_*.cf
chgrp postfix /etc/postfix/pgsql-virtual_*.cf

Add the following one line at a time. Replace qualified.hostname.com with your hostname

postconf -e 'myhostname = qualified.hostname.com'
postconf -e 'mydestination = qualified.hostname.com, localhost, localhost.localdomain'
postconf -e 'mynetworks = 127.0.0.0/8'
postconf -e 'message_size_limit = 30720000'
postconf -e 'virtual_alias_domains ='
postconf -e 'virtual_alias_maps = proxy:pgsql:/etc/postfix/pgsql-virtual_forwardings.cf, pgsql:/etc/postfix/pgsql-virtual_email2email.cf'
postconf -e 'virtual_mailbox_domains = proxy:pgsql:/etc/postfix/pgsql-virtual_domains.cf'
postconf -e 'virtual_mailbox_maps = proxy:pgsql:/etc/postfix/pgsql-virtual_mailboxes.cf'
postconf -e 'virtual_mailbox_base = /home/vmail'postconf -e 'virtual_uid_maps = static:5000'postconf -e 'virtual_gid_maps = static:5000'
postconf -e 'smtpd_sasl_auth_enable = yes'postconf -e 'broken_sasl_auth_clients = yes'
postconf -e 'smtpd_sasl_authenticated_header = yes'
postconf -e 'smtpd_recipient_restrictions = permit_mynetworks, permit_sasl_authenticated, reject_unauth_destination'
postconf -e 'smtpd_use_tls = yes'
postconf -e 'smtpd_tls_cert_file = /etc/postfix/smtpd.cert'
postconf -e 'smtpd_tls_key_file = /etc/postfix/smtpd.key'
postconf -e 'virtual_create_maildirsize = yes'
postconf -e 'virtual_maildir_extended = yes'
postconf -e 'proxy_read_maps = $local_recipient_maps $mydestination $virtual_alias_maps $virtual_alias_domains $virtual_mailbox_maps $virtual_mailbox_domains $relay_recipient_maps $relay_domains $canonical_maps $sender_canonical_maps $recipient_canonical_maps $relocated_maps $transport_maps $mynetworks $virtual_mailbox_limit_maps'
postconf -e virtual_transport=dovecotpostconf -e dovecot_destination_recipient_limit=1

Change postfix listening port to 2525 in /etc/postfix/master.cf.

2525      inet  n       -       -       -       -       smtpd

Or you could just uncomment the submission line and default to port 587.

submission inet n       -       -       -       -       smtpd

Configure saslauthd to use Postgres

Create file /etc/pam.d/smtp. Replace your_password with your own password.

auth    required   pam_pgsql.so user=mail_admin passwd=your_password host=127.0.0.1 port=6789 db=mail table=users usercolumn=email passwdcolumn=password crypt=1
account sufficient pam_pgsql.so user=mail_admin passwd=your_password host=127.0.0.1 port=6789 db=mail table=users usercolumn=email passwdcolumn=password crypt=1

Create file /etc/postfix/sasl/smtpd.conf. Replace your_password with your own password.

pwcheck_method: saslauthd
mech_list: plain login
allow_plaintext: true
auxprop_plugin: sql
sql_engine: pgsql
sql_hostnames: 127.0.0.1:6789
sql_user: mail_admin
sql_passwd: your_password
sql_database: mail
sql_select: select password from users where email = '%u'

Create file /etc/pam_pgsql.conf. Replace your_password with your own password.

database = mail
host = localhost
user = mail_admin
password = your_password
table = users
user_column = email
pwd_column = password
pw_type = crypt

Set proper permissions to files.

chmod o= /etc/pam_pgsql.conf
chmod o= /etc/postfix/sasl/smtpd.conf

Configure Dovecot

One thing that was pointed out at the end of the guide by user Nathan  is to change cmusieve for sieve on file /etc/dovecot/dovecot.conf on this following part

protocol lda {
    log_path = /home/vmail/dovecot-deliver.log
    auth_socket_path = /var/run/dovecot/auth-master
    postmaster_address = postmaster@example.com
    mail_plugins = cmusieve #change to sieve
    global_script_path = /home/vmail/globalsieverc
}

Create file /etc/dovecot/dovecot-sql.conf. Replace your_password with your own password.

driver = pgsql
connect = host=127.0.0.1 port=6789 dbname=mail user=mail_admin password=your_password
default_pass_scheme = CRYPT
password_query = SELECT email as user, password FROM users WHERE email='%u';


And thats about it. If you managed to configure everything correctly congratulations!, if not post a comment and we’ll try to find the solution.

Troubleshooting

Remember that /var/log/syslog and /var/log/mail.log are your friends. You can also add the word debug at the end of each line on /etc/pam.d/smtp to help you troubleshoot any problems you may encounter.

 
by webjunkie on DB, Pylons, Python, SQLAlchemy, uWSGI

repoze.what with reflected tables using SQLAlchemy in Pylons

One of the many things I like about SQLAlchemy is the feature of reflected tables, this means I can develop dashboards for existing applications with my current favorite framework(Pylons). So I had to create an authorization mechanism for my dashboard and this recipe from the Pylons cookbook was just what I needed.
The only thing I had to change was the way my tables were being mapped since the users table schema was already defined. This is what I did:

user_table = schema.Table('users', Base.metadata, autoload=True, autoload_with=engine)
user_group_table = schema.Table('user_groups',Base.metadata,autoload=True,autoload_with=engine)
group_permission_table = schema.Table('group_permission',Base.metadata,autoload=True,autoload_with=engine)
permission_table = schema.Table('permissions', Base.metadata,autoload=True,autoload_with=engine)
group_table = schema.Table('groups', Base.metadata,autoload=True,autoload_with=engine)
orm.mapper(User,user_table, properties={
      'groups':orm.relation(Group, secondary=user_group_table),
    })
orm.mapper(Permission,permission_table,properties={
      'groups':orm.relation(Group, secondary=group_permission_table),
    })
orm.mapper(Group,group_table,properties={
      'permissions':orm.relation(Permission, secondary=group_permission_table),
      'users':orm.relation(User, secondary=user_group_table),
    })
orm.mapper(UGT,user_group_table)
orm.mapper(GPT,group_permission_table)

And here’s the schema I had to create, except for the users table:

CREATE TABLE user_groups( 
id serial NOT NULL,  
user_id integer,  
group_id integer,  
CONSTRAINT user_groups_pkey PRIMARY KEY (id), 
CONSTRAINT gid_fkey FOREIGN KEY (group_id) REFERENCES groups (id)
CONSTRAINT uid_fkey FOREIGN KEY (user_id)   REFERENCES users (id)
)

CREATE TABLE group_permission(
id serial NOT NULL,
gid integer,
permid integer,
CONSTRAINT group_permission_pkey PRIMARY KEY (id),
CONSTRAINT groupid_fkey FOREIGN KEY (gid) REFERENCES groups (id),
CONSTRAINT permid_fkey FOREIGN KEY (permid) REFERENCES permissions (id)
)

CREATE TABLE permissions(
  id serial NOT NULL,
  pname character varying(50),
  CONSTRAINT permissions_pkey PRIMARY KEY (id)
)

CREATE TABLE groups(
  id serial NOT NULL,
  gname character varying(50),
  CONSTRAINT groups_pkey PRIMARY KEY (id)
)

CREATE TABLE users(
id serial NOT NULL,
username character varying(50),
email character varying(50),
upassword character varying(150),
CONSTRAINT users_pkey PRIMARY KEY (id)
)

And that’s all there’s to it. I was very surprise on how flexible SQLAlchemy is.