<commit_before>#!/usr/bin/python2
# -*- coding: utf-8 -*-
########################################################################
#
#  Odoo Tools by CLEARCORP S.A.
#  Copyright (C) 2009-TODAY CLEARCORP S.A. (<http://clearcorp.co.cr>).
#
#  This program is free software: you can redistribute it and/or modify
#  it under the terms of the GNU Affero General Public License as
#  published by the Free Software Foundation, either version 3 of the
#  License, or (at your option) any later version.
#
#  This program is distributed in the hope that it will be useful, but
#  WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
#  Affero General Public License for more details.
#
#  You should have received a copy of the GNU Affero General Public
#  License along with this program.  If not, see
#  <http://www.gnu.org/licenses/>.
#
########################################################################

import os, datetime, pwd, grp, getpass, re, tempfile
import logging
from odootools.lib import config, git_lib, tools, apache, phppgadmin

_logger = logging.getLogger('odootools.odoo.server')

class odooServer(object):
    
    def __init__(self, instance=None):
        self._os_info = tools.get_os()
        self._postgresql_version = ''
        if self._os_info['os'] == 'Linux' and self._os_info['version'][0] == 'Ubuntu':
            #Support for previous LTS
            if self._os_info['version'][1] == '12.04':
                self._postgresql_version = '9.1'
            #Support for versions between both LTS version
            elif self._os_info['version'][1] < '14.04':
                self._postgresql_version = '9.1'
            #Support for current LTS
            elif self._os_info['version'][1] == '14.04':
                self._postgresql_version = '9.3'
            else:
                self._postgresql_version = '9.3' #This should fail unsupported version
        # TODO check versions for Linux Mint and Arch Linux
        elif self._os_info['os'] == 'Linux' and self._os_info['version'][0] == 'LinuxMint':
            self._postgresql_version = '9.1'
        elif self._os_info['os'] == 'Linux' and self._os_info['version'][0] == 'arch':
            self._postgresql_version = '9.1'

        # TODO check if related to bazaar
        if instance:
            _logger.debug('Server initialized with instance: %s' % instance)
            _logger.debug('Branch: %s' % instance._branch)
            self._branch = instance._branch
        else:
            _logger.debug('Server initialized without instance: %s' % instance)
            self._branch = config.params['branch'] or '8.0'
        
        self._installation_type = config.params[self._branch+'_installation_type'] or config.params['installation_type'] or 'dev'
        self._user = config.params[self._branch+'_user'] or config.params['user'] or None

        if 'install_odoo_clearcorp' in config.params:
            self._install_odoo_clearcorp = config.params['install_odoo_clearcorp']
        elif self._branch+'_install_odoo_clearcorp' in config.params:
            self._install_odoo_clearcorp = config.params[self._branch+'_install_odoo_clearcorp']
        else:
            self._install_odoo_clearcorp = True

        if 'install_odoo_costa_rica' in config.params:
            self._install_odoo_costa_rica = config.params['install_odoo_costa_rica']
        elif self._branch+'_install_odoo_costa_rica' in config.params:
            self._install_odoo_costa_rica = config.params[self._branch+'_install_odoo_costa_rica']
        else:
            self._install_odoo_costa_rica = True

        if self._branch+'_update_postgres_hba' in config.params:
            self._update_postgres_hba = config.params[self._branch+'_update_postgres_hba']
        elif 'update_postgres_hba' in config.params:
            self._update_postgres_hba = config.params['update_postgres_hba']
        else:
            self._update_postgres_hba = True

        if self._branch+'_create_postgres_user' in config.params:
            self._create_postgres_user = config.params[self._branch+'_create_postgres_user']
        elif 'create_postgres_user' in config.params:
            self._create_postgres_user = config.params['create_postgres_user']
        else:
            self._create_postgres_user = True

        if self._branch+'_install_apache' in config.params:
            self._install_apache = config.params[self._branch+'_install_apache']
        elif 'install_apache' in config.params:
            self._install_apache = config.params['install_apache']
        else:
            self._install_apache = True

        self._admin_password = config.params[self._branch+'_admin_password'] or config.params['admin_password'] or None
        self._postgresql_password = config.params[self._branch+'_postgresql_password'] or config.params['postgresql_password'] or None
        return super(odooServer, self).__init__()

    def _add_odoo_user(self):
        try:
            group = grp.getgrnam('odoo')
        except:
            _logger.info('odoo group doesn\'t exist, creating group.')
            # TODO: no addgroup in arch
            tools.exec_command('addgroup odoo', as_root=True)
            group = False
        else:
            _logger.debug('odoo group already exists.')

        try:
            pw = pwd.getpwnam(self._user)
        except:
            _logger.info('Creating user: (%s)' % self._user)
            tools.exec_command('adduser --system --home /var/run/odoo --no-create-home --ingroup odoo %s' % self._user, as_root=True)
        else:
            _logger.info('User %s already exists, adding to odoo group.' % self._user)
            tools.exec_command('adduser %s odoo' % self._user, as_root=True)

        return True

    #Todo check packages
    def get_packages_distro_match(self, distro):
        match = {
            'ubuntu': {
                'ghostscript':              'ghostscript',
                'graphviz':                 'graphviz',
                'libjs-mochikit':           'libjs-mochikit',
                'libjs-mootools':           'libjs-mootools',
                'python-beaker':            'python-beaker',
                'python-cherrypy3':         'python-cherrypy3',
                'python-dateutil':          'python-dateutil',
                'python-docutils':          'python-docutils',
                'python-egenix':            'python-egenix-mxdatetime',
                'python-feedparser':        'python-feedparser',
                'python-formencode':        'python-formencode',
                'python-gdata':             'python-gdata',
                'python-imaging':           'python-imaging',
                'python-jinja2':            'python-jinja2',
                'python-ldap':              'python-ldap',
                'python-libxslt1':          'python-libxslt1',
                'python-lxml':              'python-lxml',
                'python-mako':              'python-mako',
                'python-matplotlib':        'python-matplotlib',
                'python-mock':              'python-mock',
                'python-openid':            'python-openid',
                'python-openssl':           'python-openssl',
                'python-psutil':            'python-psutil',
                'python-psycopg2':          'python-psycopg2',
                'python-pybabel':           'python-pybabel',
                'python-pychart':           'python-pychart',
                'python-pydot':             'python-pydot',
                'python-pyparsing':         'python-pyparsing',
                'python-reportlab':         'python-reportlab',
                'python-setuptools':        'python-setuptools',
                'python-simplejson':        'python-simplejson',
                'python-tz':                'python-tz',
                'python-unittest2':         'python-unittest2',
                'python-vatnumber':         'python-vatnumber',
                'python-vobject':           'python-vobject',
                'python-webdav':            'python-webdav',
                'python-werkzeug':          'python-werkzeug',
                'python-xlwt':              'python-xlwt',
                'python-yaml':              'python-yaml',
                'python-zsi':               'python-zsi',
                'tinymce':                  'tinymce',
                'wget':                     'wget',
                'poppler-utils':            'poppler-utils'
            },

            'arch': {
                'repo': {
                    'ghostscript':              'ghostscript',
                    'graphviz':                 'graphviz',
                    'libjs-mochikit':           None,
                    'libjs-mootools':           'pycow',
                    'python-beaker':            'python2-beaker',
                    'python-cherrypy3':         'python2-cherrypy',
                    'python-dateutil':          'python2-dateutil',
                    'python-docutils':          'python2-docutils',
                    'python-egenix':            'python2-egenix-mx-base',
                    'python-feedparser':        'python2-feedparser',
                    'python-formencode':        'python2-formencode',
                    'python-gdata':             'python2-gdata',
                    'python-imaging':           'python2-imaging',
                    'python-jinja2':            'python2-jinja',
                    'python-ldap':              'python2-ldap',
                    'python-libxslt1':          'python2-lxml',
                    'python-lxml':              'python2-lxml',
                    'python-mako':              'python2-mako',
                    'python-matplotlib':        'python2-matplotlib',
                    'python-openssl':           'python2-pyopenssl',
                    'python-psutil':            'python2-psutil',
                    'python-psycopg2':          'python2-psycopg2',
                    'python-pybabel':           'python2-babel',
                    'python-pychart':           'python2-pychart',
                    'python-pyparsing':         'python2-pyparsing',
                    'python-reportlab':         'python2-reportlab',
                    'python-setuptools':        'python2-distribute',
                    'python-simplejson':        'python2-simplejson',
                    'python-tz':                'python2-pytz',
                    'python-vobject':           'python2-vobject',
                    'python-werkzeug':          'python2-werkzeug',
                    'python-yaml':              'python2-yaml',
                    'tinymce':                  None,
                    'wget':                     'wget',
                    'poppler-utils':            'poppler-utils' # Needs review on arch
                },
                'aur': {
                    'python-mock':              'python2-mock',
                    'python-openid':            'python2-openid',
                    'python-pydot':             'pydot',
                    'python-unittest2':         'python2-unittest2',
                    'python-vatnumber':         'python2-vatnumber',
                    'python-webdav':            'python2-pywebdav',
                    'python-xlwt':              'python2-xlwt',
                    'python-zsi':               'zsi',
                }
            }
        }

        if distro in match:
            return match[distro]
        else:
            return False

    def _install_python_libs(self):
        _logger.info('Installing the required packages and python libraries for Odoo.')

        packages = []

        # Packages need by the installer
        packages.append('python-setuptools')
        packages.append('wget')

        # Packages for 7.0
        if self._branch == '7.0':
            packages += [
                # Dependencies
                'python-dateutil',
                'python-docutils',
                'python-feedparser',
                'python-gdata',         #Google data parser
                'python-jinja2',
                'python-ldap',
                'python-libxslt1',      #Excel spreadsheets
                'python-lxml',
                'python-mako',
                'python-mock',          #For testing
                'python-openid',
                'python-psutil',
                'python-psycopg2',
                'python-pybabel',
                'python-pychart',
                'python-pydot',
                'python-pyparsing',
                'python-reportlab',
                'python-simplejson',
                'python-tz',
                'python-unittest2',     #For testing
                'python-vatnumber',
                'python-vobject',
                'python-webdav',        #For document-webdav
                'python-werkzeug',
                'python-xlwt',          #Excel spreadsheets
                'python-yaml',
                'python-zsi',

                # Recomended
                'graphviz',
                'ghostscript',
                'python-imaging',
                'python-matplotlib',
                'poppler-utils',
            ]
            
            # Packages for 8.0
        if self._branch == '8.0' or self._branch == 'trunk':
            packages += [
                # Dependencies
                'python-dateutil',
                'python-docutils',
                'python-feedparser',
                'python-gdata',         #Google data parser
                'python-jinja2',
                'python-ldap',
                'python-libxslt1',      #Excel spreadsheets
                'python-lxml',
                'python-mako',
                'python-mock',          #For testing
                'python-openid',
                'python-psutil',
                'python-psycopg2',
                'python-pybabel',
                'python-pychart',
                'python-pydot',
                'python-pyparsing',
                'python-reportlab',
                'python-simplejson',
                'python-tz',
                'python-unittest2',     #For testing
                'python-vatnumber',
                'python-vobject',
                'python-webdav',        #For document-webdav
                'python-werkzeug',
                'python-xlwt',          #Excel spreadsheets
                'python-yaml',
                'python-zsi',

                # Recomended
                'graphviz',
                'ghostscript',
                'python-imaging',
                'python-matplotlib',
                'poppler-utils',
            ]

        # Test distro and call appropriate function
        if self._os_info and self._os_info['os'] == 'Linux':
            if (self._os_info['version'][0] == 'Ubuntu') or (self._os_info['version'][0] == 'LinuxMint'):
                return self._ubuntu_install_python_libs(packages)
            elif self._os_info['version'][0] == 'arch':
                return self._arch_install_python_libs(packages)
        _logger.error('Can\'t install python libraries in this OS: %s. Exiting.' % self._os_info['version'][0])
        return False

    def _ubuntu_install_python_libs(self, packages):
        #Initialize packages match
        distro_match_packages = self.get_packages_distro_match('ubuntu')

        #Build definitive package list
        distro_packages = []
        for package in packages:
            if package in distro_match_packages:
                if distro_match_packages[package]:
                    distro_packages.append(distro_match_packages[package])
                #No package for this distro available
                else:
                    _logger.warning('No installable package for %s in Ubuntu. Please install it manually.' % package)
            else:
                _logger.warning('Package unknown: %s. Exiting.' % package)

        # Install packages
        return tools.ubuntu_install_package(distro_packages)

    def _arch_install_python_libs(self, packages):
        #Initialize packages match
        distro_match_packages = self.get_packages_distro_match('arch')

        #Build definitive package list
        distro_packages = []
        aur_packages = []
        for package in packages:
            if package in distro_match_packages['repo']:
                if distro_match_packages['repo'][package]:
                    distro_packages.append(distro_match_packages['repo'][package])
                #No package for this distro available
                else:
                    _logger.warning('No installable package for %s in Arch. Please install it manually.' % package)
            elif package in distro_match_packages['aur']:
                if distro_match_packages['aur'][package]:
                    aur_packages.append(distro_match_packages['aur'][package])
                #No package for this distro available
                else:
                    _logger.warning('No installable package for %s in Arch. Please install it manually.' % package)
            else:
                _logger.warning('Package unknown: %s. Exiting.' % package)

        # Install packages
        _logger.info('Installing packages with pacman.')
        _logger.debug('Packages: %s' % str(distro_packages))
        error = False
        if not tools.arch_install_repo_package(distro_packages):
            _logger.warning('Failed to install some repo packages. This can render the OpenERP installation unusable.')
            error = True

        _logger.info('Installing packages from AUR.')
        _logger.debug('Packages: %s' % str(aur_packages))
        if not tools.arch_install_repo_package(aur_packages):
            _logger.warning('Failed to install some AUR packages. This can render the OpenERP installation unusable.')
            error = True

        return not error

    def _install_postgresql(self):
        _logger.info('Installing PostgreSQL.')
        # Test distro and call appropriate function
        if self._os_info and self._os_info['os'] == 'Linux':
            if (self._os_info['version'][0] == 'Ubuntu') or (self._os_info['version'][0] == 'LinuxMint'):
                return self._ubuntu_install_postgresql()
            elif self._os_info['version'][0] == 'arch':
                return self._arch_install_postgresql()
        _logger.error('Can\'t install python libraries in this OS: %s. Exiting.' % self._os_info['version'][0])
        return False

    def _ubuntu_install_postgresql(self):
        return tools.ubuntu_install_package(['postgresql'])

    def _arch_install_postgresql(self):
        #TODO: change some of this calls with a python way to do it (because of perms)
        if os.path.isdir('/var/lib/postgres/data'):
            _logger.info('PostgreSQL appears to be already configured. Skipping.')
            return False

        if tools.arch_install_package(['postgresql']):
            _logger.error('Failed to install PostgreSQL. Exiting.')
            return False
        if tools.exec_command('systemd-tmpfiles --create postgresql.conf', as_root=True):
            _logger.error('Failed to configure PostgreSQL systemd script. Exiting.')
            return False
        if tools.exec_command('mkdir /var/lib/postgres/data', as_root=True):
            _logger.error('Failed to create PostgreSQL data directory. Exiting.')
            return False
        if tools.exec_command('chown -c postgres:postgres /var/lib/postgres/data', as_root=True):
            _logger.error('Failed to set PostgreSQL data directory permisions. Exiting.')
            return False
        if tools.exec_command('sudo -i -u postgres initdb -D \'/var/lib/postgres/data\''):
            _logger.error('Failed to init PostgreSQL database. Exiting.')
            return False
        if tools.exec_command('systemctl enable postgresql', as_root=True):
            _logger.error('Failed to enable PostgreSQL init script. Exiting.')
            return False
        if tools.exec_command('systemctl start postgresql', as_root=True):
            _logger.error('Failed to start PostgreSQL. Exiting.')
            return False
        return True

    def _do_update_postgres_hba(self):
        _logger.info('Updating PostgreSQL pg_hba.conf file.')
        # Test distro and call appropriate function
        if self._os_info and self._os_info['os'] == 'Linux':
            if (self._os_info['version'][0] == 'Ubuntu') or (self._os_info['version'][0] == 'LinuxMint'):
                return self._ubuntu_do_update_postgres_hba()
            elif self._os_info['version'][0] == 'arch':
                return self._arch_do_update_postgres_hba()
        _logger.error('Can\'t update PostgreSQL pg_hba.conf in this OS: %s. Exiting.' % self._os_info['version'][0])
        return False

    def _ubuntu_do_update_postgres_hba(self):
        #TODO: change sed command with python lib to do the change (be carefull with the user perms)
        if tools.exec_command("sed -i 's/\(local[[:space:]]*all[[:space:]]*all[[:space:]]*\)\(ident[[:space:]]*sameuser\)/\1md5/g' /etc/postgresql/%s/main/pg_hba.conf" % self._postgresql_version, as_root=True):
            _logger.error('Failed to set PostgreSQL pg_hba.conf file. Exiting.')
            return False
        if tools.exec_command('service postgresql restart', as_root=True):
            _logger.error('Failed to start PostgreSQL. Exiting.')
            return False
        return True

    def _arch_do_update_postgres_hba(self):
        #TODO: lp:1133383 change sed command with python lib to do the change (be carefull with the user perms)
        #if tools.exec_command("sed -i 's/\(local[[:space:]]*all[[:space:]]*all[[:space:]]*\)\(ident[[:space:]]*sameuser\)/\1md5/g' /etc/postgresql/%s/main/pg_hba.conf" % self._postgresql_version, as_root=True):
        #    _logger.error('Failed to set PostgreSQL pg_hba.conf file. Exiting.')
        #    return False
        #if tools.exec_command('/etc/init.d/postgresql%s restart' % self._postgresql_init_suffix, as_root=True)
        #    _logger.error('Failed to start PostgreSQL. Exiting.')
        #    return False
        return True

    def _add_postgresql_user(self):
        _logger.info('Adding PostgreSQL user: %s.' % self._user)
        if tools.exec_command('sudo -u postgres createuser %s --superuser --createdb --no-createrole' % self._user):
            _logger.error('Failed to add PostgreSQL user. Exiting.')
            return False
        if tools.exec_command('sudo -u postgres psql template1 -U postgres -c "alter user \\"%s\\" with password \'%s\'"' % (self._user, self._admin_password)):
            _logger.error('Failed to set PostgreSQL user password. Exiting.')
            return False
        return True

    def _change_postgresql_admin_password(self):
        _logger.info('Changing PostgreSQL admin password.')
        if tools.exec_command('sudo -u postgres psql template1 -U postgres -c "alter user postgres with password \'%s\'"' % self._postgresql_password):
            _logger.error('Failed to set PostgreSQL admin password. Exiting.')
            return False
        return True

    def change_perms(self):
        _logger.debug('User: %s' % self._user)
        if os.path.isdir('/srv/odoo'):
            if tools.exec_command('chown -R %s:odoo /srv/odoo' % self._user, as_root=True):
                _logger.warning('Failed to set /srv/odoo owner. Skipping.')
            if tools.exec_command('chmod -R g+w /srv/odoo', as_root=True):
                _logger.warning('Failed to set /srv/odoo perms. Skipping.')
        if os.path.isdir('/var/log/odoo'):
            if tools.exec_command('chown -R %s:odoo /var/log/odoo' % self._user, as_root=True):
                _logger.warning('Failed to set /var/log/odoo owner. Skipping.')
            if tools.exec_command('chmod -R g+w /var/log/odoo', as_root=True):
                _logger.warning('Failed to set /var/log/odoo perms. Skipping.')
        if os.path.isdir('/var/run/odoo'):
            if tools.exec_command('chown -R %s:odoo /var/run/odoo' % self._user, as_root=True):
                _logger.warning('Failed to set /var/run/odoo owner. Skipping.')
            if tools.exec_command('chmod -R g+w /var/run/odoo', as_root=True):
                _logger.warning('Failed to set /var/run/odoo perms. Skipping.')
        if os.path.isdir('/etc/odoo'):
            if tools.exec_command('chown -R %s:odoo /etc/odoo' % self._user, as_root=True):
                _logger.warning('Failed to set /etc/odoo owner. Skipping.')

        return True

    def _download_odoo_repo(self):
        if os.path.isdir('/srv/odoo'):
            _logger.warning('/srv/odoo already exists. Not downloading repo.')
            return False

        _logger.info('Creating the directory structure.')
        if tools.exec_command('mkdir -p /srv/odoo/%s/instances' % self._branch, as_root=True):
            _logger.error('Failed to create the odoo directory structure. Exiting.')
            return False
        if tools.exec_command('mkdir -p /srv/odoo/%s/src' % self._branch, as_root=True):
            _logger.error('Failed to create the odoo src directory structure. Exiting.')
            return False
        cwd = os.getcwd()
        os.chdir('/srv/odoo/%s/src/' % self._branch)
        _logger.info('Cloning latest Odoo git repository.')
        repo = git.git_clone('<EMAIL>:CLEARCORP/odoo.git', 'odoo', branch=self._branch) # TODO: select source
        os.chdir(cwd)
        _logger.info('Cloning finished.')
        return True

    def _download_git_repo(self, source, name):
        if not os.path.isdir('/srv/odoo/%s/src/' % self._branch):
            _logger.warning('/srv/odoo/%s/src/ do not exists. Not cloning repo.' % self._branch)
            return False
        cwd = os.getcwd()
        os.chdir('/srv/odoo/%s/src/' % self._branch)
        _logger.info('Cloning from %s latest %s branch.' % (source, self._branch))
        repo = git.git_clone(source, name, branch=self._branch)
        os.chdir(cwd)
        _logger.info('Cloning finished.')
        return True

    def _download_odoo(self, modules_to_install):
        _logger.info('Downloading latest Odoo %s release.' % self._branch)
        if os.path.isdir('/srv/odoo'):
            _logger.info('Odoo repo already exists.')
            repo_downloaded = True
        else:
            repo_downloaded = self._download_odoo_repo()

        self.change_perms()

        if 'odoo-clearcorp' in modules_to_install and modules_to_install['odoo-clearcorp']:
            self._download_git_repo('<EMAIL>:CLEARCORP/odoo-clearcorp.git', 'odoo-clearcorp')
        if 'odoo-costa-rica' in modules_to_install and modules_to_install['odoo-costa-rica']:
            self._download_git_repo('<EMAIL>:CLEARCORP/odoo-costa-rica.git', 'odoo-costa-rica')
        self.change_perms()
        return True

    def _config_odoo_version(self):
        _logger.info('Configuring Odoo %s' % self._branch)
        cwd = os.getcwd()

        # Odoo Server bin
        _logger.debug('Copy bin script skeleton to /etc')
        if tools.exec_command('mkdir -p /etc/odoo/%s/server/' % self._branch, as_root=True):
            _logger.error('Failed to make /etc/odoo/%s/server/ directory. Exiting.' % self._branch)
            return False
        if tools.exec_command('cp %s/odootools/odoo/static/bin/server-bin-%s-skeleton /etc/odoo/%s/server/bin-skeleton' % (config.params['odootools_path'], self._branch, self._branch), as_root=True):
            _logger.error('Failed to copy bin skeleton. Exiting.')<commit_msg>[FIX] - Renamed git_lib calls and protocol from git clone.<commit_after>#!/usr/bin/python2
# -*- coding: utf-8 -*-
########################################################################
#
#  Odoo Tools by CLEARCORP S.A.
#  Copyright (C) 2009-TODAY CLEARCORP S.A. (<http://clearcorp.co.cr>).
#
#  This program is free software: you can redistribute it and/or modify
#  it under the terms of the GNU Affero General Public License as
#  published by the Free Software Foundation, either version 3 of the
#  License, or (at your option) any later version.
#
#  This program is distributed in the hope that it will be useful, but
#  WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
#  Affero General Public License for more details.
#
#  You should have received a copy of the GNU Affero General Public
#  License along with this program.  If not, see
#  <http://www.gnu.org/licenses/>.
#
########################################################################

import os, datetime, pwd, grp, getpass, re, tempfile
import logging
from odootools.lib import config, git_lib, tools, apache, phppgadmin

_logger = logging.getLogger('odootools.odoo.server')

class odooServer(object):
    
    def __init__(self, instance=None):
        self._os_info = tools.get_os()
        self._postgresql_version = ''
        if self._os_info['os'] == 'Linux' and self._os_info['version'][0] == 'Ubuntu':
            #Support for previous LTS
            if self._os_info['version'][1] == '12.04':
                self._postgresql_version = '9.1'
            #Support for versions between both LTS version
            elif self._os_info['version'][1] < '14.04':
                self._postgresql_version = '9.1'
            #Support for current LTS
            elif self._os_info['version'][1] == '14.04':
                self._postgresql_version = '9.3'
            else:
                self._postgresql_version = '9.3' #This should fail unsupported version
        # TODO check versions for Linux Mint and Arch Linux
        elif self._os_info['os'] == 'Linux' and self._os_info['version'][0] == 'LinuxMint':
            self._postgresql_version = '9.1'
        elif self._os_info['os'] == 'Linux' and self._os_info['version'][0] == 'arch':
            self._postgresql_version = '9.1'

        # TODO check if related to bazaar
        if instance:
            _logger.debug('Server initialized with instance: %s' % instance)
            _logger.debug('Branch: %s' % instance._branch)
            self._branch = instance._branch
        else:
            _logger.debug('Server initialized without instance: %s' % instance)
            self._branch = config.params['branch'] or '8.0'
        
        self._installation_type = config.params[self._branch+'_installation_type'] or config.params['installation_type'] or 'dev'
        self._user = config.params[self._branch+'_user'] or config.params['user'] or None

        if 'install_odoo_clearcorp' in config.params:
            self._install_odoo_clearcorp = config.params['install_odoo_clearcorp']
        elif self._branch+'_install_odoo_clearcorp' in config.params:
            self._install_odoo_clearcorp = config.params[self._branch+'_install_odoo_clearcorp']
        else:
            self._install_odoo_clearcorp = True

        if 'install_odoo_costa_rica' in config.params:
            self._install_odoo_costa_rica = config.params['install_odoo_costa_rica']
        elif self._branch+'_install_odoo_costa_rica' in config.params:
            self._install_odoo_costa_rica = config.params[self._branch+'_install_odoo_costa_rica']
        else:
            self._install_odoo_costa_rica = True

        if self._branch+'_update_postgres_hba' in config.params:
            self._update_postgres_hba = config.params[self._branch+'_update_postgres_hba']
        elif 'update_postgres_hba' in config.params:
            self._update_postgres_hba = config.params['update_postgres_hba']
        else:
            self._update_postgres_hba = True

        if self._branch+'_create_postgres_user' in config.params:
            self._create_postgres_user = config.params[self._branch+'_create_postgres_user']
        elif 'create_postgres_user' in config.params:
            self._create_postgres_user = config.params['create_postgres_user']
        else:
            self._create_postgres_user = True

        if self._branch+'_install_apache' in config.params:
            self._install_apache = config.params[self._branch+'_install_apache']
        elif 'install_apache' in config.params:
            self._install_apache = config.params['install_apache']
        else:
            self._install_apache = True

        self._admin_password = config.params[self._branch+'_admin_password'] or config.params['admin_password'] or None
        self._postgresql_password = config.params[self._branch+'_postgresql_password'] or config.params['postgresql_password'] or None
        return super(odooServer, self).__init__()

    def _add_odoo_user(self):
        try:
            group = grp.getgrnam('odoo')
        except:
            _logger.info('odoo group doesn\'t exist, creating group.')
            # TODO: no addgroup in arch
            tools.exec_command('addgroup odoo', as_root=True)
            group = False
        else:
            _logger.debug('odoo group already exists.')

        try:
            pw = pwd.getpwnam(self._user)
        except:
            _logger.info('Creating user: (%s)' % self._user)
            tools.exec_command('adduser --system --home /var/run/odoo --no-create-home --ingroup odoo %s' % self._user, as_root=True)
        else:
            _logger.info('User %s already exists, adding to odoo group.' % self._user)
            tools.exec_command('adduser %s odoo' % self._user, as_root=True)

        return True

    #Todo check packages
    def get_packages_distro_match(self, distro):
        match = {
            'ubuntu': {
                'ghostscript':              'ghostscript',
                'graphviz':                 'graphviz',
                'libjs-mochikit':           'libjs-mochikit',
                'libjs-mootools':           'libjs-mootools',
                'python-beaker':            'python-beaker',
                'python-cherrypy3':         'python-cherrypy3',
                'python-dateutil':          'python-dateutil',
                'python-docutils':          'python-docutils',
                'python-egenix':            'python-egenix-mxdatetime',
                'python-feedparser':        'python-feedparser',
                'python-formencode':        'python-formencode',
                'python-gdata':             'python-gdata',
                'python-imaging':           'python-imaging',
                'python-jinja2':            'python-jinja2',
                'python-ldap':              'python-ldap',
                'python-libxslt1':          'python-libxslt1',
                'python-lxml':              'python-lxml',
                'python-mako':              'python-mako',
                'python-matplotlib':        'python-matplotlib',
                'python-mock':              'python-mock',
                'python-openid':            'python-openid',
                'python-openssl':           'python-openssl',
                'python-psutil':            'python-psutil',
                'python-psycopg2':          'python-psycopg2',
                'python-pybabel':           'python-pybabel',
                'python-pychart':           'python-pychart',
                'python-pydot':             'python-pydot',
                'python-pyparsing':         'python-pyparsing',
                'python-reportlab':         'python-reportlab',
                'python-setuptools':        'python-setuptools',
                'python-simplejson':        'python-simplejson',
                'python-tz':                'python-tz',
                'python-unittest2':         'python-unittest2',
                'python-vatnumber':         'python-vatnumber',
                'python-vobject':           'python-vobject',
                'python-webdav':            'python-webdav',
                'python-werkzeug':          'python-werkzeug',
                'python-xlwt':              'python-xlwt',
                'python-yaml':              'python-yaml',
                'python-zsi':               'python-zsi',
                'tinymce':                  'tinymce',
                'wget':                     'wget',
                'poppler-utils':            'poppler-utils'
            },

            'arch': {
                'repo': {
                    'ghostscript':              'ghostscript',
                    'graphviz':                 'graphviz',
                    'libjs-mochikit':           None,
                    'libjs-mootools':           'pycow',
                    'python-beaker':            'python2-beaker',
                    'python-cherrypy3':         'python2-cherrypy',
                    'python-dateutil':          'python2-dateutil',
                    'python-docutils':          'python2-docutils',
                    'python-egenix':            'python2-egenix-mx-base',
                    'python-feedparser':        'python2-feedparser',
                    'python-formencode':        'python2-formencode',
                    'python-gdata':             'python2-gdata',
                    'python-imaging':           'python2-imaging',
                    'python-jinja2':            'python2-jinja',
                    'python-ldap':              'python2-ldap',
                    'python-libxslt1':          'python2-lxml',
                    'python-lxml':              'python2-lxml',
                    'python-mako':              'python2-mako',
                    'python-matplotlib':        'python2-matplotlib',
                    'python-openssl':           'python2-pyopenssl',
                    'python-psutil':            'python2-psutil',
                    'python-psycopg2':          'python2-psycopg2',
                    'python-pybabel':           'python2-babel',
                    'python-pychart':           'python2-pychart',
                    'python-pyparsing':         'python2-pyparsing',
                    'python-reportlab':         'python2-reportlab',
                    'python-setuptools':        'python2-distribute',
                    'python-simplejson':        'python2-simplejson',
                    'python-tz':                'python2-pytz',
                    'python-vobject':           'python2-vobject',
                    'python-werkzeug':          'python2-werkzeug',
                    'python-yaml':              'python2-yaml',
                    'tinymce':                  None,
                    'wget':                     'wget',
                    'poppler-utils':            'poppler-utils' # Needs review on arch
                },
                'aur': {
                    'python-mock':              'python2-mock',
                    'python-openid':            'python2-openid',
                    'python-pydot':             'pydot',
                    'python-unittest2':         'python2-unittest2',
                    'python-vatnumber':         'python2-vatnumber',
                    'python-webdav':            'python2-pywebdav',
                    'python-xlwt':              'python2-xlwt',
                    'python-zsi':               'zsi',
                }
            }
        }

        if distro in match:
            return match[distro]
        else:
            return False

    def _install_python_libs(self):
        _logger.info('Installing the required packages and python libraries for Odoo.')

        packages = []

        # Packages need by the installer
        packages.append('python-setuptools')
        packages.append('wget')

        # Packages for 7.0
        if self._branch == '7.0':
            packages += [
                # Dependencies
                'python-dateutil',
                'python-docutils',
                'python-feedparser',
                'python-gdata',         #Google data parser
                'python-jinja2',
                'python-ldap',
                'python-libxslt1',      #Excel spreadsheets
                'python-lxml',
                'python-mako',
                'python-mock',          #For testing
                'python-openid',
                'python-psutil',
                'python-psycopg2',
                'python-pybabel',
                'python-pychart',
                'python-pydot',
                'python-pyparsing',
                'python-reportlab',
                'python-simplejson',
                'python-tz',
                'python-unittest2',     #For testing
                'python-vatnumber',
                'python-vobject',
                'python-webdav',        #For document-webdav
                'python-werkzeug',
                'python-xlwt',          #Excel spreadsheets
                'python-yaml',
                'python-zsi',

                # Recomended
                'graphviz',
                'ghostscript',
                'python-imaging',
                'python-matplotlib',
                'poppler-utils',
            ]
            
            # Packages for 8.0
        if self._branch == '8.0' or self._branch == 'trunk':
            packages += [
                # Dependencies
                'python-dateutil',
                'python-docutils',
                'python-feedparser',
                'python-gdata',         #Google data parser
                'python-jinja2',
                'python-ldap',
                'python-libxslt1',      #Excel spreadsheets
                'python-lxml',
                'python-mako',
                'python-mock',          #For testing
                'python-openid',
                'python-psutil',
                'python-psycopg2',
                'python-pybabel',
                'python-pychart',
                'python-pydot',
                'python-pyparsing',
                'python-reportlab',
                'python-simplejson',
                'python-tz',
                'python-unittest2',     #For testing
                'python-vatnumber',
                'python-vobject',
                'python-webdav',        #For document-webdav
                'python-werkzeug',
                'python-xlwt',          #Excel spreadsheets
                'python-yaml',
                'python-zsi',

                # Recomended
                'graphviz',
                'ghostscript',
                'python-imaging',
                'python-matplotlib',
                'poppler-utils',
            ]

        # Test distro and call appropriate function
        if self._os_info and self._os_info['os'] == 'Linux':
            if (self._os_info['version'][0] == 'Ubuntu') or (self._os_info['version'][0] == 'LinuxMint'):
                return self._ubuntu_install_python_libs(packages)
            elif self._os_info['version'][0] == 'arch':
                return self._arch_install_python_libs(packages)
        _logger.error('Can\'t install python libraries in this OS: %s. Exiting.' % self._os_info['version'][0])
        return False

    def _ubuntu_install_python_libs(self, packages):
        #Initialize packages match
        distro_match_packages = self.get_packages_distro_match('ubuntu')

        #Build definitive package list
        distro_packages = []
        for package in packages:
            if package in distro_match_packages:
                if distro_match_packages[package]:
                    distro_packages.append(distro_match_packages[package])
                #No package for this distro available
                else:
                    _logger.warning('No installable package for %s in Ubuntu. Please install it manually.' % package)
            else:
                _logger.warning('Package unknown: %s. Exiting.' % package)

        # Install packages
        return tools.ubuntu_install_package(distro_packages)

    def _arch_install_python_libs(self, packages):
        #Initialize packages match
        distro_match_packages = self.get_packages_distro_match('arch')

        #Build definitive package list
        distro_packages = []
        aur_packages = []
        for package in packages:
            if package in distro_match_packages['repo']:
                if distro_match_packages['repo'][package]:
                    distro_packages.append(distro_match_packages['repo'][package])
                #No package for this distro available
                else:
                    _logger.warning('No installable package for %s in Arch. Please install it manually.' % package)
            elif package in distro_match_packages['aur']:
                if distro_match_packages['aur'][package]:
                    aur_packages.append(distro_match_packages['aur'][package])
                #No package for this distro available
                else:
                    _logger.warning('No installable package for %s in Arch. Please install it manually.' % package)
            else:
                _logger.warning('Package unknown: %s. Exiting.' % package)

        # Install packages
        _logger.info('Installing packages with pacman.')
        _logger.debug('Packages: %s' % str(distro_packages))
        error = False
        if not tools.arch_install_repo_package(distro_packages):
            _logger.warning('Failed to install some repo packages. This can render the OpenERP installation unusable.')
            error = True

        _logger.info('Installing packages from AUR.')
        _logger.debug('Packages: %s' % str(aur_packages))
        if not tools.arch_install_repo_package(aur_packages):
            _logger.warning('Failed to install some AUR packages. This can render the OpenERP installation unusable.')
            error = True

        return not error

    def _install_postgresql(self):
        _logger.info('Installing PostgreSQL.')
        # Test distro and call appropriate function
        if self._os_info and self._os_info['os'] == 'Linux':
            if (self._os_info['version'][0] == 'Ubuntu') or (self._os_info['version'][0] == 'LinuxMint'):
                return self._ubuntu_install_postgresql()
            elif self._os_info['version'][0] == 'arch':
                return self._arch_install_postgresql()
        _logger.error('Can\'t install python libraries in this OS: %s. Exiting.' % self._os_info['version'][0])
        return False

    def _ubuntu_install_postgresql(self):
        return tools.ubuntu_install_package(['postgresql'])

    def _arch_install_postgresql(self):
        #TODO: change some of this calls with a python way to do it (because of perms)
        if os.path.isdir('/var/lib/postgres/data'):
            _logger.info('PostgreSQL appears to be already configured. Skipping.')
            return False

        if tools.arch_install_package(['postgresql']):
            _logger.error('Failed to install PostgreSQL. Exiting.')
            return False
        if tools.exec_command('systemd-tmpfiles --create postgresql.conf', as_root=True):
            _logger.error('Failed to configure PostgreSQL systemd script. Exiting.')
            return False
        if tools.exec_command('mkdir /var/lib/postgres/data', as_root=True):
            _logger.error('Failed to create PostgreSQL data directory. Exiting.')
            return False
        if tools.exec_command('chown -c postgres:postgres /var/lib/postgres/data', as_root=True):
            _logger.error('Failed to set PostgreSQL data directory permisions. Exiting.')
            return False
        if tools.exec_command('sudo -i -u postgres initdb -D \'/var/lib/postgres/data\''):
            _logger.error('Failed to init PostgreSQL database. Exiting.')
            return False
        if tools.exec_command('systemctl enable postgresql', as_root=True):
            _logger.error('Failed to enable PostgreSQL init script. Exiting.')
            return False
        if tools.exec_command('systemctl start postgresql', as_root=True):
            _logger.error('Failed to start PostgreSQL. Exiting.')
            return False
        return True

    def _do_update_postgres_hba(self):
        _logger.info('Updating PostgreSQL pg_hba.conf file.')
        # Test distro and call appropriate function
        if self._os_info and self._os_info['os'] == 'Linux':
            if (self._os_info['version'][0] == 'Ubuntu') or (self._os_info['version'][0] == 'LinuxMint'):
                return self._ubuntu_do_update_postgres_hba()
            elif self._os_info['version'][0] == 'arch':
                return self._arch_do_update_postgres_hba()
        _logger.error('Can\'t update PostgreSQL pg_hba.conf in this OS: %s. Exiting.' % self._os_info['version'][0])
        return False

    def _ubuntu_do_update_postgres_hba(self):
        #TODO: change sed command with python lib to do the change (be carefull with the user perms)
        if tools.exec_command("sed -i 's/\(local[[:space:]]*all[[:space:]]*all[[:space:]]*\)\(ident[[:space:]]*sameuser\)/\1md5/g' /etc/postgresql/%s/main/pg_hba.conf" % self._postgresql_version, as_root=True):
            _logger.error('Failed to set PostgreSQL pg_hba.conf file. Exiting.')
            return False
        if tools.exec_command('service postgresql restart', as_root=True):
            _logger.error('Failed to start PostgreSQL. Exiting.')
            return False
        return True

    def _arch_do_update_postgres_hba(self):
        #TODO: lp:1133383 change sed command with python lib to do the change (be carefull with the user perms)
        #if tools.exec_command("sed -i 's/\(local[[:space:]]*all[[:space:]]*all[[:space:]]*\)\(ident[[:space:]]*sameuser\)/\1md5/g' /etc/postgresql/%s/main/pg_hba.conf" % self._postgresql_version, as_root=True):
        #    _logger.error('Failed to set PostgreSQL pg_hba.conf file. Exiting.')
        #    return False
        #if tools.exec_command('/etc/init.d/postgresql%s restart' % self._postgresql_init_suffix, as_root=True)
        #    _logger.error('Failed to start PostgreSQL. Exiting.')
        #    return False
        return True

    def _add_postgresql_user(self):
        _logger.info('Adding PostgreSQL user: %s.' % self._user)
        if tools.exec_command('sudo -u postgres createuser %s --superuser --createdb --no-createrole' % self._user):
            _logger.error('Failed to add PostgreSQL user. Exiting.')
            return False
        if tools.exec_command('sudo -u postgres psql template1 -U postgres -c "alter user \\"%s\\" with password \'%s\'"' % (self._user, self._admin_password)):
            _logger.error('Failed to set PostgreSQL user password. Exiting.')
            return False
        return True

    def _change_postgresql_admin_password(self):
        _logger.info('Changing PostgreSQL admin password.')
        if tools.exec_command('sudo -u postgres psql template1 -U postgres -c "alter user postgres with password \'%s\'"' % self._postgresql_password):
            _logger.error('Failed to set PostgreSQL admin password. Exiting.')
            return False
        return True

    def change_perms(self):
        _logger.debug('User: %s' % self._user)
        if os.path.isdir('/srv/odoo'):
            if tools.exec_command('chown -R %s:odoo /srv/odoo' % self._user, as_root=True):
                _logger.warning('Failed to set /srv/odoo owner. Skipping.')
            if tools.exec_command('chmod -R g+w /srv/odoo', as_root=True):
                _logger.warning('Failed to set /srv/odoo perms. Skipping.')
        if os.path.isdir('/var/log/odoo'):
            if tools.exec_command('chown -R %s:odoo /var/log/odoo' % self._user, as_root=True):
                _logger.warning('Failed to set /var/log/odoo owner. Skipping.')
            if tools.exec_command('chmod -R g+w /var/log/odoo', as_root=True):
                _logger.warning('Failed to set /var/log/odoo perms. Skipping.')
        if os.path.isdir('/var/run/odoo'):
            if tools.exec_command('chown -R %s:odoo /var/run/odoo' % self._user, as_root=True):
                _logger.warning('Failed to set /var/run/odoo owner. Skipping.')
            if tools.exec_command('chmod -R g+w /var/run/odoo', as_root=True):
                _logger.warning('Failed to set /var/run/odoo perms. Skipping.')
        if os.path.isdir('/etc/odoo'):
            if tools.exec_command('chown -R %s:odoo /etc/odoo' % self._user, as_root=True):
                _logger.warning('Failed to set /etc/odoo owner. Skipping.')

        return True

    def _download_odoo_repo(self):
        if os.path.isdir('/srv/odoo'):
            _logger.warning('/srv/odoo already exists. Not downloading repo.')
            return False

        _logger.info('Creating the directory structure.')
        if tools.exec_command('mkdir -p /srv/odoo/%s/instances' % self._branch, as_root=True):
            _logger.error('Failed to create the odoo directory structure. Exiting.')
            return False
        if tools.exec_command('mkdir -p /srv/odoo/%s/src' % self._branch, as_root=True):
            _logger.error('Failed to create the odoo src directory structure. Exiting.')
            return False
        cwd = os.getcwd()
        os.chdir('/srv/odoo/%s/src/' % self._branch)
        _logger.info('Cloning the latest Odoo git repository from https://github.com/CLEARCORP/odoo.git latest %s branch.' % self._branch))
        repo = git_lib.git_clone('https://github.com/CLEARCORP/odoo.git', 'odoo', branch=self._branch)
        os.chdir(cwd)
        _logger.info('Cloning finished.')
        return True

    def _download_git_repo(self, source, name):
        if not os.path.isdir('/srv/odoo/%s/src/' % self._branch):
            _logger.warning('/srv/odoo/%s/src/ do not exists. Not cloning repo.' % self._branch)
            return False
        cwd = os.getcwd()
        os.chdir('/srv/odoo/%s/src/' % self._branch)
        _logger.info('Cloning from %s latest %s branch.' % (source, self._branch))
        repo = git_lib.git_clone(source, name, branch=self._branch)
        os.chdir(cwd)
        _logger.info('Cloning finished.')
        return True

    def _download_odoo(self, modules_to_install):
        _logger.info('Downloading latest Odoo %s release.' % self._branch)
        if os.path.isdir('/srv/odoo'):
            _logger.info('Odoo repo already exists.')
            repo_downloaded = True
        else:
            repo_downloaded = self._download_odoo_repo()

        self.change_perms()

        if 'odoo-clearcorp' in modules_to_install and modules_to_install['odoo-clearcorp']:
            self._download_git_repo('https://github.com/CLEARCORP/odoo-clearcorp.git', 'odoo-clearcorp')
        if 'odoo-costa-rica' in modules_to_install and modules_to_install['odoo-costa-rica']:
            self._download_git_repo('https://github.com/CLEARCORP/odoo-costa-rica.git', 'odoo-costa-rica')
        self.change_perms()
        return True

    def _config_odoo_version(self):
        _logger.info('Configuring Odoo %s' % self._branch)
        cwd = os.getcwd()

        # Odoo Server bin
        _logger.debug('Copy bin script skeleton to /etc')
        if tools.exec_command('mkdir -p /etc/odoo/%s/server/' % self._branch, as_root=True):
            _logger.error('Failed to make /etc/odoo/%s/server/ directory. Exiting.' % self._branch)
            return False
        if tools.exec_command('cp %s/odootools/odoo/static/bin/server-bin-%s-skeleton /etc/odoo/%s/server/bin-skeleton' % (config.params['odootools_path'], self._branch, self._branch), as_root=True):
            _logger.error('Failed to copy bin skeleton. Exiting.')
{'fraction_non_alphanumeric': 0.11917704658362734, 'fraction_numerical': 0.0047159622723018215, 'mean_word_length': 1.8268966569989316, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 10, 'https://': 4, 'lorem ipsum': 0, 'www.': 2, 'xml': 14}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 20, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '738594', 'n_tokens_mistral': 16245, 'n_tokens_neox': 15529, 'n_words': 3313}