<commit_before># -*- coding: utf-8 -*-

import ast
import base64
import csv
import glob
import itertools
import logging
import operator
import datetime
import hashlib
import os
import re
import simplejson
import time
import urllib2
import xmlrpclib
import zlib
from xml.etree import ElementTree
from cStringIO import StringIO

import babel.messages.pofile
import werkzeug.utils
import werkzeug.wrappers
try:
    import xlwt
except ImportError:
    xlwt = None

from .. import common
openerpweb = common.http

#----------------------------------------------------------
# OpenERP Web helpers
#----------------------------------------------------------

def sass2scss(src):
    # Validated by diff -u of sass2scss against:
    # sass-convert -F sass -T scss openerp.sass openerp.scss
    block = []
    sass = ('', block)
    reComment = re.compile(r'//.*$')
    reIndent = re.compile(r'^\s+')
    reIgnore = re.compile(r'^\s*(//.*)?$')
    reFixes = { re.compile(r'\(\((.*)\)\)') : r'(\1)', }
    lastLevel = 0
    prevBlocks = {}
    for l in src.split('\n'):
        l = l.rstrip()
        if reIgnore.search(l): continue
        l = reComment.sub('', l)
        l = l.rstrip()
        indent = reIndent.match(l)
        level = indent.end() if indent else 0
        l = l[level:]
        if level>lastLevel:
            prevBlocks[lastLevel] = block
            newBlock = []
            block[-1] = (block[-1], newBlock)
            block = newBlock
        elif level<lastLevel:
            block = prevBlocks[level]
        lastLevel = level
        if not l: continue
        # Fixes
        for ereg, repl in reFixes.items():
            l = ereg.sub(repl if type(repl)==str else repl(), l)
        block.append(l)

    def write(sass, level=-1):
        out = ""
        indent = '  '*level
        if type(sass)==tuple:
            if level>=0:
                out += indent+sass[0]+" {\n"
            for e in sass[1]:
                out += write(e, level+1)
            if level>=0:
                out = out.rstrip(" \n")
                out += ' }\n'
            if level==0:
                out += "\n"
        else:
            out += indent+sass+";\n"
        return out
    return write(sass)

def db_list(req):
    proxy = req.session.proxy("db")
    dbs = proxy.list()
    h = req.httprequest.environ['HTTP_HOST'].split(':')[0]
    d = h.split('.')[0]
    r = req.config.dbfilter.replace('%h', h).replace('%d', d)
    dbs = [i for i in dbs if re.match(r, i)]
    return dbs

def module_topological_sort(modules):
    """ Return a list of module names sorted so that their dependencies of the
    modules are listed before the module itself

    modules is a dict of {module_name: dependencies}

    :param modules: modules to sort
    :type modules: dict
    :returns: list(str)
    """

    dependencies = set(itertools.chain.from_iterable(modules.itervalues()))
    # incoming edge: dependency on other module (if a depends on b, a has an
    # incoming edge from b, aka there's an edge from b to a)
    # outgoing edge: other module depending on this one

    # [Tarjan 1976], http://en.wikipedia.org/wiki/Topological_sorting#Algorithms
    #L ← Empty list that will contain the sorted nodes
    L = []
    #S ← Set of all nodes with no outgoing edges (modules on which no other
    #    module depends)
    S = set(module for module in modules if module not in dependencies)

    visited = set()
    #function visit(node n)
    def visit(n):
        #if n has not been visited yet then
        if n not in visited:
            #mark n as visited
            visited.add(n)
            #change: n not web module, can not be resolved, ignore
            if n not in modules: return
            #for each node m with an edge from m to n do (dependencies of n)
            for m in modules[n]:
                #visit(m)
                visit(m)
            #add n to L
            L.append(n)
    #for each node n in S do
    for n in S:
        #visit(n)
        visit(n)
    return L

def module_installed(req):
    # Candidates module the current heuristic is the /static dir
    loadable = openerpweb.addons_manifest.keys()
    modules = {}

    # Retrieve database installed modules
    # TODO The following code should move to ir.module.module.list_installed_modules()
    Modules = req.session.model('ir.module.module')
    domain = [('state','=','installed'), ('name','in', loadable)]
    for module in Modules.search_read(domain, ['name', 'dependencies_id']):
        modules[module['name']] = []
        deps = module.get('dependencies_id')
        if deps:
            deps_read = req.session.model('ir.module.module.dependency').read(deps, ['name'])
            dependencies = [i['name'] for i in deps_read]
            modules[module['name']] = dependencies

    sorted_modules = module_topological_sort(modules)
    return sorted_modules

def module_installed_bypass_session(dbname):
    loadable = openerpweb.addons_manifest.keys()
    modules = {}
    try:
        import openerp.modules.registry
        registry = openerp.modules.registry.RegistryManager.get(dbname)
        cr = registry.db.cursor()
        try:
            m = registry.get('ir.module.module')
            # TODO The following code should move to ir.module.module.list_installed_modules()
            domain = [('state','=','installed'), ('name','in', loadable)]
            ids = m.search(cr, 1, [('state','=','installed'), ('name','in', loadable)])
            for module in m.read(cr, 1, ids, ['name', 'dependencies_id']):
                modules[module['name']] = []
                deps = module.get('dependencies_id')
                if deps:
                    deps_read = registry.get('ir.module.module.dependency').read(cr, 1, deps, ['name'])
                    dependencies = [i['name'] for i in deps_read]
                    modules[module['name']] = dependencies
        finally:
            cr.close()
    except Exception,e:
        pass
    sorted_modules = module_topological_sort(modules)
    return sorted_modules

def module_boot(req):
    dbs = db_list(req)
    serverside = []
    dbside = []<commit_msg>[IMP] use registry contextmanager<commit_after># -*- coding: utf-8 -*-

import ast
import base64
import csv
import glob
import itertools
import logging
import operator
import datetime
import hashlib
import os
import re
import simplejson
import time
import urllib2
import xmlrpclib
import zlib
from xml.etree import ElementTree
from cStringIO import StringIO

import babel.messages.pofile
import werkzeug.utils
import werkzeug.wrappers
try:
    import xlwt
except ImportError:
    xlwt = None

from .. import common
openerpweb = common.http

#----------------------------------------------------------
# OpenERP Web helpers
#----------------------------------------------------------

def sass2scss(src):
    # Validated by diff -u of sass2scss against:
    # sass-convert -F sass -T scss openerp.sass openerp.scss
    block = []
    sass = ('', block)
    reComment = re.compile(r'//.*$')
    reIndent = re.compile(r'^\s+')
    reIgnore = re.compile(r'^\s*(//.*)?$')
    reFixes = { re.compile(r'\(\((.*)\)\)') : r'(\1)', }
    lastLevel = 0
    prevBlocks = {}
    for l in src.split('\n'):
        l = l.rstrip()
        if reIgnore.search(l): continue
        l = reComment.sub('', l)
        l = l.rstrip()
        indent = reIndent.match(l)
        level = indent.end() if indent else 0
        l = l[level:]
        if level>lastLevel:
            prevBlocks[lastLevel] = block
            newBlock = []
            block[-1] = (block[-1], newBlock)
            block = newBlock
        elif level<lastLevel:
            block = prevBlocks[level]
        lastLevel = level
        if not l: continue
        # Fixes
        for ereg, repl in reFixes.items():
            l = ereg.sub(repl if type(repl)==str else repl(), l)
        block.append(l)

    def write(sass, level=-1):
        out = ""
        indent = '  '*level
        if type(sass)==tuple:
            if level>=0:
                out += indent+sass[0]+" {\n"
            for e in sass[1]:
                out += write(e, level+1)
            if level>=0:
                out = out.rstrip(" \n")
                out += ' }\n'
            if level==0:
                out += "\n"
        else:
            out += indent+sass+";\n"
        return out
    return write(sass)

def db_list(req):
    proxy = req.session.proxy("db")
    dbs = proxy.list()
    h = req.httprequest.environ['HTTP_HOST'].split(':')[0]
    d = h.split('.')[0]
    r = req.config.dbfilter.replace('%h', h).replace('%d', d)
    dbs = [i for i in dbs if re.match(r, i)]
    return dbs

def module_topological_sort(modules):
    """ Return a list of module names sorted so that their dependencies of the
    modules are listed before the module itself

    modules is a dict of {module_name: dependencies}

    :param modules: modules to sort
    :type modules: dict
    :returns: list(str)
    """

    dependencies = set(itertools.chain.from_iterable(modules.itervalues()))
    # incoming edge: dependency on other module (if a depends on b, a has an
    # incoming edge from b, aka there's an edge from b to a)
    # outgoing edge: other module depending on this one

    # [Tarjan 1976], http://en.wikipedia.org/wiki/Topological_sorting#Algorithms
    #L ← Empty list that will contain the sorted nodes
    L = []
    #S ← Set of all nodes with no outgoing edges (modules on which no other
    #    module depends)
    S = set(module for module in modules if module not in dependencies)

    visited = set()
    #function visit(node n)
    def visit(n):
        #if n has not been visited yet then
        if n not in visited:
            #mark n as visited
            visited.add(n)
            #change: n not web module, can not be resolved, ignore
            if n not in modules: return
            #for each node m with an edge from m to n do (dependencies of n)
            for m in modules[n]:
                #visit(m)
                visit(m)
            #add n to L
            L.append(n)
    #for each node n in S do
    for n in S:
        #visit(n)
        visit(n)
    return L

def module_installed(req):
    # Candidates module the current heuristic is the /static dir
    loadable = openerpweb.addons_manifest.keys()
    modules = {}

    # Retrieve database installed modules
    # TODO The following code should move to ir.module.module.list_installed_modules()
    Modules = req.session.model('ir.module.module')
    domain = [('state','=','installed'), ('name','in', loadable)]
    for module in Modules.search_read(domain, ['name', 'dependencies_id']):
        modules[module['name']] = []
        deps = module.get('dependencies_id')
        if deps:
            deps_read = req.session.model('ir.module.module.dependency').read(deps, ['name'])
            dependencies = [i['name'] for i in deps_read]
            modules[module['name']] = dependencies

    sorted_modules = module_topological_sort(modules)
    return sorted_modules

def module_installed_bypass_session(dbname):
    loadable = openerpweb.addons_manifest.keys()
    modules = {}
    try:
        import openerp.modules.registry
        registry = openerp.modules.registry.RegistryManager.get(dbname)
        with registry.cursor() as cr:
            m = registry.get('ir.module.module')
            # TODO The following code should move to ir.module.module.list_installed_modules()
            domain = [('state','=','installed'), ('name','in', loadable)]
            ids = m.search(cr, 1, [('state','=','installed'), ('name','in', loadable)])
            for module in m.read(cr, 1, ids, ['name', 'dependencies_id']):
                modules[module['name']] = []
                deps = module.get('dependencies_id')
                if deps:
                    deps_read = registry.get('ir.module.module.dependency').read(cr, 1, deps, ['name'])
                    dependencies = [i['name'] for i in deps_read]
                    modules[module['name']] = dependencies
    except Exception,e:
        pass
    sorted_modules = module_topological_sort(modules)
    return sorted_modules

def module_boot(req):
    dbs = db_list(req)
    serverside = []
    dbside = []
{'fraction_non_alphanumeric': 0.1387141573400866, 'fraction_numerical': 0.004411404297034556, 'mean_word_length': 2.1134282807731433, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3602623', 'n_tokens_mistral': 3751, 'n_tokens_neox': 3640, 'n_words': 1096}