<commit_before># -*- encoding: utf-8 -*-
##############################################################################
#
# Copyright (c) 2004-2008 TINY SPRL. (http://tiny.be)
#
# $Id$
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsability of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# garantees and support are strongly adviced to contact a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
#
##############################################################################

import os, sys, imp
import itertools
from sets import Set

import osv
import tools
import pooler


import netsvc
from osv import fields

import zipfile

logger = netsvc.Logger()

opj = os.path.join

_ad = os.path.abspath(opj(tools.config['root_path'], 'addons'))     # default addons path (base)
ad = os.path.abspath(tools.config['addons_path'])           # alternate addons path

sys.path.insert(1, _ad)
if ad != _ad:
    sys.path.insert(1, ad)

class Graph(dict):

    def addNode(self, name, deps):
        max_depth, father = 0, None
        for n in [Node(x, self) for x in deps]:
            if n.depth >= max_depth:
                father = n
                max_depth = n.depth
        if father:
            father.addChild(name)
        else:
            Node(name, self)

    def __iter__(self):
        level = 0
        done = Set(self.keys())
        while done:
            level_modules = [(name, module) for name, module in self.items() if module.depth==level]
            for name, module in level_modules:
                done.remove(name)
                yield module
            level += 1

class Singleton(object):

    def __new__(cls, name, graph):
        if name in graph:
            inst = graph[name]
        else:
            inst = object.__new__(cls)
            inst.name = name
            graph[name] = inst
        return inst

class Node(Singleton):

    def __init__(self, name, graph):
        self.graph = graph
        if not hasattr(self, 'childs'):
            self.childs = []
        if not hasattr(self, 'depth'):
            self.depth = 0

    def addChild(self, name):
        node = Node(name, self.graph)
        node.depth = self.depth + 1
        if node not in self.childs:
            self.childs.append(node)
        for attr in ('init', 'update', 'demo'):
            if hasattr(self, attr):
                setattr(node, attr, True)
        self.childs.sort(lambda x,y: cmp(x.name, y.name))

    def hasChild(self, name):
        return Node(name, self.graph) in self.childs or \
                bool([c for c in self.childs if c.hasChild(name)])

    def __setattr__(self, name, value):
        super(Singleton, self).__setattr__(name, value)
        if name in ('init', 'update', 'demo'):
            tools.config[name][self.name] = 1
            for child in self.childs:
                setattr(child, name, value)
        if name == 'depth':
            for child in self.childs:
                setattr(child, name, value + 1)

    def __iter__(self):
        return itertools.chain(iter(self.childs), *map(iter, self.childs))

    def __str__(self):
        return self._pprint()

    def _pprint(self, depth=0):
        s = '%s\n' % self.name
        for c in self.childs:
            s += '%s`-> %s' % ('   ' * depth, c._pprint(depth+1))
        return s

def get_module_path(module):
    """Return the path of the given module.
    """

    if os.path.exists(opj(ad, module)) or os.path.exists(opj(ad, '%s.zip' % module)):
        return opj(ad, module)

    if os.path.exists(opj(_ad, module)) or os.path.exists(opj(_ad, '%s.zip' % module)):
        return opj(_ad, module)

    raise IOError, 'Module not found : %s' % module

def get_module_resource(module, *args):
    """Return the full path of a resource of the given module.

    @param module: the module
    @param args: the resource path components

    @return: absolute path to the resource
    """
    return opj(get_module_path(module), *args)

def get_modules():
    """Returns the list of module names
    """

    module_list = os.listdir(ad)
    module_names = [os.path.basename(m) for m in module_list]
    module_list += [m for m in os.listdir(_ad) if m not in module_names]

    return module_list

def create_graph(module_list, force=None):
    if not force:
        force=[]
    graph = Graph()
    packages = []

    for module in module_list:
        if module[-4:]=='.zip':
            module = module[:-4]
        mod_path = get_module_path(module)
        terp_file = get_module_resource(module, '__terp__.py')
        if os.path.isfile(terp_file) or zipfile.is_zipfile(mod_path):
            try:
                info = eval(tools.file_open(terp_file).read())
            except:
                logger.notifyChannel('init', netsvc.LOG_ERROR, 'addon:%s:eval file %s' % (module, terp_file))
                raise
            if info.get('installable', True):
                packages.append((module, info.get('depends', []), info))

    current,later = Set([p for p, dep, data in packages]), Set()
    while packages and current > later:
        package, deps, datas = packages[0]

        # if all dependencies of 'package' are already in the graph, add 'package' in the graph
        if reduce(lambda x,y: x and y in graph, deps, True):
            if not package in current:
                packages.pop(0)
                continue
            later.clear()
            current.remove(package)
            graph.addNode(package, deps)
            node = Node(package, graph)
            node.datas = datas
            for kind in ('init', 'demo', 'update'):
                if package in tools.config[kind] or 'all' in tools.config[kind] or kind in force:
                    setattr(node, kind, True)
        else:
            later.add(package)
            packages.append((package, deps, datas))
        packages.pop(0)
    
    for package in later:
        logger.notifyChannel('init', netsvc.LOG_ERROR, 'addon:%s:Unmet dependency' % package)

    return graph

def init_module_objects(cr, module_name, obj_list):
    pool = pooler.get_pool(cr.dbname)
    logger.notifyChannel('init', netsvc.LOG_INFO, 'addon:%s:creating or updating database tables' % module_name)
    for obj in obj_list:
        if hasattr(obj, 'init'):
            obj.init(cr)
        obj._auto_init(cr, {'module': module_name})
        cr.commit()

def load_module_graph(cr, graph, status=None, **kwargs):
    # **kwargs is passed directly to convert_xml_import
    if not status:
        status={}

    status = status.copy()
    package_todo = []
    statusi = 0
    for package in graph:
        status['progress'] = (float(statusi)+0.1)/len(graph)
        m = package.name
        logger.notifyChannel('init', netsvc.LOG_INFO, 'addon:%s' % m)
        sys.stdout.flush()
        pool = pooler.get_pool(cr.dbname)
        modules = pool.instanciate(m, cr)
        cr.execute('select state, demo from ir_module_module where name=%s', (m,))
        (package_state, package_demo) = (cr.rowcount and cr.fetchone()) or ('uninstalled', False)
        idref = {}
        status['progress'] = (float(statusi)+0.4)/len(graph)
        if hasattr(package, 'init') or hasattr(package, 'update') or package_state in ('to install', 'to upgrade'):
            init_module_objects(cr, m, modules)
            for kind in ('init', 'update'):
                for filename in package.datas.get('%s_xml' % kind, []):
                    mode = 'update'
                    if hasattr(package, 'init') or package_state=='to install':
                        mode = 'init'
                    logger.notifyChannel('init', netsvc.LOG_INFO, 'addon:%s:loading %s' % (m, filename))
                    name, ext = os.path.splitext(filename)
                    if ext == '.csv':
                        tools.convert_csv_import(cr, m, os.path.basename(filename), tools.file_open(opj(m, filename)).read(), idref, mode=mode)
                    elif ext == '.sql':
                        queries = tools.file_open(opj(m, filename)).read().split(';')
                        for query in queries:
                            new_query = ' '.join(query.split())
                            if new_query:
                                cr.execute(new_query)
                    else:
                        tools.convert_xml_import(cr, m, tools.file_open(opj(m, filename)), idref, mode=mode, **kwargs)
            if hasattr(package, 'demo') or (package_demo and package_state != 'installed'):
                status['progress'] = (float(statusi)+0.75)/len(graph)
                for xml in package.datas.get('demo_xml', []):
                    name, ext = os.path.splitext(xml)
                    logger.notifyChannel('init', netsvc.LOG_INFO, 'addon:%s:loading %s' % (m, xml))
                    if ext == '.csv':
                        tools.convert_csv_import(cr, m, os.path.basename(xml), tools.file_open(opj(m, xml)).read(), idref, noupdate=True)
                    else:
                        tools.convert_xml_import(cr, m, tools.file_open(opj(m, xml)), idref, noupdate=True, **kwargs)
                cr.execute('update ir_module_module set demo=%s where name=%s', (True, package.name))
            package_todo.append(package.name)
            cr.execute("update ir_module_module set state='installed' where state in ('to upgrade', 'to install') and name=%s", (package.name,))
        cr.commit()
        statusi+=1

    pool = pooler.get_pool(cr.dbname)
    cr.execute('select * from ir_model where state=%s', ('manual',))
    for model in cr.dictfetchall():
        pool.get('ir.model').instanciate(cr, 1, model['model'], {})

    pool.get('ir.model.data')._process_end(cr, 1, package_todo)
    cr.commit()

def register_classes():
    module_list = get_modules()
    for package in create_graph(module_list):
        m = package.name
        logger.notifyChannel('init', netsvc.LOG_INFO, 'addon:%s:registering classes' % m)
        sys.stdout.flush()

        mod_path = get_module_path(m)
        if not os.path.isfile(mod_path+'.zip'):
            # XXX must restrict to only addons paths
            imp.load_module(m, *imp.find_module(m))
        else:
            import zipimport
            try:
                zimp = zipimport.zipimporter(mod_path+'.zip')<commit_msg>during addons initialisation, warn user if the objects doesn't specify access rules<commit_after># -*- encoding: utf-8 -*-
##############################################################################
#
# Copyright (c) 2004-2008 TINY SPRL. (http://tiny.be)
#
# $Id$
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsability of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# garantees and support are strongly adviced to contact a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
#
##############################################################################

import os, sys, imp
import itertools
from sets import Set

import osv
import tools
import pooler


import netsvc
from osv import fields

import zipfile

logger = netsvc.Logger()

opj = os.path.join

_ad = os.path.abspath(opj(tools.config['root_path'], 'addons'))     # default addons path (base)
ad = os.path.abspath(tools.config['addons_path'])           # alternate addons path

sys.path.insert(1, _ad)
if ad != _ad:
    sys.path.insert(1, ad)

class Graph(dict):

    def addNode(self, name, deps):
        max_depth, father = 0, None
        for n in [Node(x, self) for x in deps]:
            if n.depth >= max_depth:
                father = n
                max_depth = n.depth
        if father:
            father.addChild(name)
        else:
            Node(name, self)

    def __iter__(self):
        level = 0
        done = Set(self.keys())
        while done:
            level_modules = [(name, module) for name, module in self.items() if module.depth==level]
            for name, module in level_modules:
                done.remove(name)
                yield module
            level += 1

class Singleton(object):

    def __new__(cls, name, graph):
        if name in graph:
            inst = graph[name]
        else:
            inst = object.__new__(cls)
            inst.name = name
            graph[name] = inst
        return inst

class Node(Singleton):

    def __init__(self, name, graph):
        self.graph = graph
        if not hasattr(self, 'childs'):
            self.childs = []
        if not hasattr(self, 'depth'):
            self.depth = 0

    def addChild(self, name):
        node = Node(name, self.graph)
        node.depth = self.depth + 1
        if node not in self.childs:
            self.childs.append(node)
        for attr in ('init', 'update', 'demo'):
            if hasattr(self, attr):
                setattr(node, attr, True)
        self.childs.sort(lambda x,y: cmp(x.name, y.name))

    def hasChild(self, name):
        return Node(name, self.graph) in self.childs or \
                bool([c for c in self.childs if c.hasChild(name)])

    def __setattr__(self, name, value):
        super(Singleton, self).__setattr__(name, value)
        if name in ('init', 'update', 'demo'):
            tools.config[name][self.name] = 1
            for child in self.childs:
                setattr(child, name, value)
        if name == 'depth':
            for child in self.childs:
                setattr(child, name, value + 1)

    def __iter__(self):
        return itertools.chain(iter(self.childs), *map(iter, self.childs))

    def __str__(self):
        return self._pprint()

    def _pprint(self, depth=0):
        s = '%s\n' % self.name
        for c in self.childs:
            s += '%s`-> %s' % ('   ' * depth, c._pprint(depth+1))
        return s

def get_module_path(module):
    """Return the path of the given module.
    """

    if os.path.exists(opj(ad, module)) or os.path.exists(opj(ad, '%s.zip' % module)):
        return opj(ad, module)

    if os.path.exists(opj(_ad, module)) or os.path.exists(opj(_ad, '%s.zip' % module)):
        return opj(_ad, module)

    raise IOError, 'Module not found : %s' % module

def get_module_resource(module, *args):
    """Return the full path of a resource of the given module.

    @param module: the module
    @param args: the resource path components

    @return: absolute path to the resource
    """
    return opj(get_module_path(module), *args)

def get_modules():
    """Returns the list of module names
    """

    module_list = os.listdir(ad)
    module_names = [os.path.basename(m) for m in module_list]
    module_list += [m for m in os.listdir(_ad) if m not in module_names]

    return module_list

def create_graph(module_list, force=None):
    if not force:
        force=[]
    graph = Graph()
    packages = []

    for module in module_list:
        if module[-4:]=='.zip':
            module = module[:-4]
        mod_path = get_module_path(module)
        terp_file = get_module_resource(module, '__terp__.py')
        if os.path.isfile(terp_file) or zipfile.is_zipfile(mod_path):
            try:
                info = eval(tools.file_open(terp_file).read())
            except:
                logger.notifyChannel('init', netsvc.LOG_ERROR, 'addon:%s:eval file %s' % (module, terp_file))
                raise
            if info.get('installable', True):
                packages.append((module, info.get('depends', []), info))

    current,later = Set([p for p, dep, data in packages]), Set()
    while packages and current > later:
        package, deps, datas = packages[0]

        # if all dependencies of 'package' are already in the graph, add 'package' in the graph
        if reduce(lambda x,y: x and y in graph, deps, True):
            if not package in current:
                packages.pop(0)
                continue
            later.clear()
            current.remove(package)
            graph.addNode(package, deps)
            node = Node(package, graph)
            node.datas = datas
            for kind in ('init', 'demo', 'update'):
                if package in tools.config[kind] or 'all' in tools.config[kind] or kind in force:
                    setattr(node, kind, True)
        else:
            later.add(package)
            packages.append((package, deps, datas))
        packages.pop(0)
    
    for package in later:
        logger.notifyChannel('init', netsvc.LOG_ERROR, 'addon:%s:Unmet dependency' % package)

    return graph

def init_module_objects(cr, module_name, obj_list):
    pool = pooler.get_pool(cr.dbname)
    logger.notifyChannel('init', netsvc.LOG_INFO, 'addon:%s:creating or updating database tables' % module_name)
    for obj in obj_list:
        if hasattr(obj, 'init'):
            obj.init(cr)
        obj._auto_init(cr, {'module': module_name})
        cr.commit()

def load_module_graph(cr, graph, status=None, **kwargs):
    # **kwargs is passed directly to convert_xml_import
    if not status:
        status={}

    status = status.copy()
    package_todo = []
    statusi = 0
    for package in graph:
        status['progress'] = (float(statusi)+0.1)/len(graph)
        m = package.name
        logger.notifyChannel('init', netsvc.LOG_INFO, 'addon:%s' % m)
        sys.stdout.flush()
        pool = pooler.get_pool(cr.dbname)
        modules = pool.instanciate(m, cr)
        cr.execute('select state, demo from ir_module_module where name=%s', (m,))
        (package_state, package_demo) = (cr.rowcount and cr.fetchone()) or ('uninstalled', False)
        idref = {}
        status['progress'] = (float(statusi)+0.4)/len(graph)
        if hasattr(package, 'init') or hasattr(package, 'update') or package_state in ('to install', 'to upgrade'):
            init_module_objects(cr, m, modules)
            for kind in ('init', 'update'):
                for filename in package.datas.get('%s_xml' % kind, []):
                    mode = 'update'
                    if hasattr(package, 'init') or package_state=='to install':
                        mode = 'init'
                    logger.notifyChannel('init', netsvc.LOG_INFO, 'addon:%s:loading %s' % (m, filename))
                    name, ext = os.path.splitext(filename)
                    if ext == '.csv':
                        tools.convert_csv_import(cr, m, os.path.basename(filename), tools.file_open(opj(m, filename)).read(), idref, mode=mode)
                    elif ext == '.sql':
                        queries = tools.file_open(opj(m, filename)).read().split(';')
                        for query in queries:
                            new_query = ' '.join(query.split())
                            if new_query:
                                cr.execute(new_query)
                    else:
                        tools.convert_xml_import(cr, m, tools.file_open(opj(m, filename)), idref, mode=mode, **kwargs)
            if hasattr(package, 'demo') or (package_demo and package_state != 'installed'):
                status['progress'] = (float(statusi)+0.75)/len(graph)
                for xml in package.datas.get('demo_xml', []):
                    name, ext = os.path.splitext(xml)
                    logger.notifyChannel('init', netsvc.LOG_INFO, 'addon:%s:loading %s' % (m, xml))
                    if ext == '.csv':
                        tools.convert_csv_import(cr, m, os.path.basename(xml), tools.file_open(opj(m, xml)).read(), idref, noupdate=True)
                    else:
                        tools.convert_xml_import(cr, m, tools.file_open(opj(m, xml)), idref, noupdate=True, **kwargs)
                cr.execute('update ir_module_module set demo=%s where name=%s', (True, package.name))
            package_todo.append(package.name)
            cr.execute("update ir_module_module set state='installed' where state in ('to upgrade', 'to install') and name=%s", (package.name,))
        
        # check if all model of the module have at least a access rule.
        cr.execute("""  SELECT name 
                         FROM ir_model m 
                        WHERE EXISTS (SELECT 1 
                                        FROM ir_model_data 
                                       WHERE module = %s 
                                         AND model = m.name
                                      ) 
                          AND NOT EXISTS (SELECT 1
                                            FROM ir_model_access
                                           WHERE model_id = m.id
                                          )
                   """, (m,))

        for (model,) in cr.fetchall():
            logger.notifyChannel('init', netsvc.LOG_WARNING, 'addon:%s:object %s has no access rules!' % (m,model,))

        cr.commit()
        statusi+=1

    pool = pooler.get_pool(cr.dbname)
    cr.execute('select * from ir_model where state=%s', ('manual',))
    for model in cr.dictfetchall():
        pool.get('ir.model').instanciate(cr, 1, model['model'], {})

    pool.get('ir.model.data')._process_end(cr, 1, package_todo)
    cr.commit()

def register_classes():
    module_list = get_modules()
    for package in create_graph(module_list):
        m = package.name
        logger.notifyChannel('init', netsvc.LOG_INFO, 'addon:%s:registering classes' % m)
        sys.stdout.flush()

        mod_path = get_module_path(m)
        if not os.path.isfile(mod_path+'.zip'):
            # XXX must restrict to only addons paths
            imp.load_module(m, *imp.find_module(m))
        else:
            import zipimport
            try:
                zimp = zipimport.zipimporter(mod_path+'.zip')
{'fraction_non_alphanumeric': 0.129182476716109, 'fraction_numerical': 0.004484304932735426, 'mean_word_length': 2.1784294915718787, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 22}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1421396', 'n_tokens_mistral': 6931, 'n_tokens_neox': 6654, 'n_words': 1971}