<commit_before># Copyright (c) 2013 Mirantis Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import bisect
import collections
import copy
import time

from oslo_log import log as logging
import six

from stackalytics.processor import launchpad_utils
from stackalytics.processor import user_processor
from stackalytics.processor import utils


LOG = logging.getLogger(__name__)


class RecordProcessor(object):
    def __init__(self, runtime_storage_inst):
        self.runtime_storage_inst = runtime_storage_inst

        self.domains_index = runtime_storage_inst.get_by_key('companies')

        self.releases = runtime_storage_inst.get_by_key('releases')
        self.releases_dates = [r['end_date'] for r in self.releases]

        self.modules = None
        self.alias_module_map = None

    def _get_release(self, timestamp):
        release_index = bisect.bisect(self.releases_dates, timestamp)
        if release_index >= len(self.releases):
            LOG.warn('Timestamp %s is beyond releases boundaries, the last '
                     'release will be used. Please consider adding a '
                     'new release into default_data.json', timestamp)
            release_index = len(self.releases) - 1
        return self.releases[release_index]['release_name']

    def _get_modules(self):
        if self.modules is None:
            self.modules = set()
            self.alias_module_map = dict()

            for repo in utils.load_repos(self.runtime_storage_inst):
                module = repo['module'].lower()
                module_aliases = repo.get('aliases') or []

                add = True
                for module_name in ([module] + module_aliases):
                    for m in self.modules:
                        if module_name.find(m) >= 0:
                            add = False
                            break
                        if m.find(module_name) >= 0:
                            self.modules.remove(m)
                            break
                    if add:
                        self.modules.add(module_name)

                for alias in module_aliases:
                    self.alias_module_map[alias] = module

        return self.modules, self.alias_module_map

    def _find_company(self, companies, date):
        for r in companies:
            if date < r['end_date']:
                return r['company_name'], 'strict'
        return companies[-1]['company_name'], 'open'  # may be overridden

    def _get_company_by_email(self, email):
        if not email:
            return None

        name, at, domain = email.partition('@')
        if domain:
            parts = domain.split('.')
            for i in range(len(parts), 1, -1):
                m = '.'.join(parts[len(parts) - i:])
                if m in self.domains_index:
                    return self.domains_index[m]
        return None

    def _create_user(self, launchpad_id, email, gerrit_id, user_name):
        company = (self._get_company_by_email(email) or
                   self._get_independent())
        emails = []
        if email:
            emails = [email]
        user = {
            'user_id': user_processor.make_user_id(
                emails=emails, launchpad_id=launchpad_id, gerrit_id=gerrit_id),
            'launchpad_id': launchpad_id,
            'user_name': user_name or '',
            'companies': [{
                'company_name': company,
                'end_date': 0,
            }],
            'emails': emails,
        }
        if gerrit_id:
            user['gerrit_id'] = gerrit_id
        return user

    def _get_lp_info(self, email):
        lp_profile = None
        if not utils.check_email_validity(email):
            LOG.debug('User email is not valid %s', email)
        else:
            lp_profile = launchpad_utils.lp_profile_by_email(email)

        if not lp_profile:
            LOG.debug('User with email %s not found', email)
            return None, None

        LOG.debug('Email %(email)s is mapped to launchpad user %(lp)s',
                  {'email': email, 'lp': lp_profile['name']})
        return lp_profile['name'], lp_profile['display_name']

    def _get_lp_user_name(self, launchpad_id):
        if not launchpad_id:
            return None

        lp_profile = launchpad_utils.lp_profile_by_launchpad_id(launchpad_id)

        if not lp_profile:
            LOG.debug('User with id %s not found', launchpad_id)
            return launchpad_id

        return lp_profile['display_name']

    def _get_independent(self):
        return '*independent'

    def _update_user_affiliation(self, user):
        for email in user.get('emails'):
            company_name = self._get_company_by_email(email)
            uc = user['companies']
            if (company_name and (len(uc) == 1) and
                    (uc[0]['company_name'] == self._get_independent())):
                LOG.debug('Updating affiliation of user %s to %s',
                          user['user_id'], company_name)
                uc[0]['company_name'] = company_name
                break

    def _get_user_exact_company(self, user):
        if len(user.get('companies', [])) == 1:
            return user['companies'][0]['company_name']
        return None

    def _merge_user_profiles(self, user_profiles):
        LOG.debug('Merge profiles: %s', user_profiles)

        # check of there are more than 1 launchpad_id nor gerrit_id
        lp_ids = set(u.get('launchpad_id') for u in user_profiles
                     if u.get('launchpad_id'))
        if len(lp_ids) > 1:
            LOG.info('Ambiguous launchpad ids: %s on profiles: %s',
                     (lp_ids, user_profiles))
        g_ids = set(u.get('gerrit_id') for u in user_profiles
                    if u.get('gerrit_id'))
        if len(g_ids) > 1:
            LOG.info('Ambiguous gerrit ids: %s on profiles: %s',
                     (g_ids, user_profiles))

        merged_user = {}  # merged user profile

        # collect ordinary fields
        for key in ['seq', 'user_name', 'user_id', 'gerrit_id', 'github_id',
                    'launchpad_id', 'companies', 'static', 'ldap_id']:
            value = next((v.get(key) for v in user_profiles if v.get(key)),
                         None)
            if value:
                merged_user[key] = value

        # update user_id, prefer it to be equal to launchpad_id
        merged_user['user_id'] = (merged_user.get('launchpad_id') or<commit_msg>Fix log pattern in merge_profiles<commit_after># Copyright (c) 2013 Mirantis Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import bisect
import collections
import copy
import time

from oslo_log import log as logging
import six

from stackalytics.processor import launchpad_utils
from stackalytics.processor import user_processor
from stackalytics.processor import utils


LOG = logging.getLogger(__name__)


class RecordProcessor(object):
    def __init__(self, runtime_storage_inst):
        self.runtime_storage_inst = runtime_storage_inst

        self.domains_index = runtime_storage_inst.get_by_key('companies')

        self.releases = runtime_storage_inst.get_by_key('releases')
        self.releases_dates = [r['end_date'] for r in self.releases]

        self.modules = None
        self.alias_module_map = None

    def _get_release(self, timestamp):
        release_index = bisect.bisect(self.releases_dates, timestamp)
        if release_index >= len(self.releases):
            LOG.warn('Timestamp %s is beyond releases boundaries, the last '
                     'release will be used. Please consider adding a '
                     'new release into default_data.json', timestamp)
            release_index = len(self.releases) - 1
        return self.releases[release_index]['release_name']

    def _get_modules(self):
        if self.modules is None:
            self.modules = set()
            self.alias_module_map = dict()

            for repo in utils.load_repos(self.runtime_storage_inst):
                module = repo['module'].lower()
                module_aliases = repo.get('aliases') or []

                add = True
                for module_name in ([module] + module_aliases):
                    for m in self.modules:
                        if module_name.find(m) >= 0:
                            add = False
                            break
                        if m.find(module_name) >= 0:
                            self.modules.remove(m)
                            break
                    if add:
                        self.modules.add(module_name)

                for alias in module_aliases:
                    self.alias_module_map[alias] = module

        return self.modules, self.alias_module_map

    def _find_company(self, companies, date):
        for r in companies:
            if date < r['end_date']:
                return r['company_name'], 'strict'
        return companies[-1]['company_name'], 'open'  # may be overridden

    def _get_company_by_email(self, email):
        if not email:
            return None

        name, at, domain = email.partition('@')
        if domain:
            parts = domain.split('.')
            for i in range(len(parts), 1, -1):
                m = '.'.join(parts[len(parts) - i:])
                if m in self.domains_index:
                    return self.domains_index[m]
        return None

    def _create_user(self, launchpad_id, email, gerrit_id, user_name):
        company = (self._get_company_by_email(email) or
                   self._get_independent())
        emails = []
        if email:
            emails = [email]
        user = {
            'user_id': user_processor.make_user_id(
                emails=emails, launchpad_id=launchpad_id, gerrit_id=gerrit_id),
            'launchpad_id': launchpad_id,
            'user_name': user_name or '',
            'companies': [{
                'company_name': company,
                'end_date': 0,
            }],
            'emails': emails,
        }
        if gerrit_id:
            user['gerrit_id'] = gerrit_id
        return user

    def _get_lp_info(self, email):
        lp_profile = None
        if not utils.check_email_validity(email):
            LOG.debug('User email is not valid %s', email)
        else:
            lp_profile = launchpad_utils.lp_profile_by_email(email)

        if not lp_profile:
            LOG.debug('User with email %s not found', email)
            return None, None

        LOG.debug('Email %(email)s is mapped to launchpad user %(lp)s',
                  {'email': email, 'lp': lp_profile['name']})
        return lp_profile['name'], lp_profile['display_name']

    def _get_lp_user_name(self, launchpad_id):
        if not launchpad_id:
            return None

        lp_profile = launchpad_utils.lp_profile_by_launchpad_id(launchpad_id)

        if not lp_profile:
            LOG.debug('User with id %s not found', launchpad_id)
            return launchpad_id

        return lp_profile['display_name']

    def _get_independent(self):
        return '*independent'

    def _update_user_affiliation(self, user):
        for email in user.get('emails'):
            company_name = self._get_company_by_email(email)
            uc = user['companies']
            if (company_name and (len(uc) == 1) and
                    (uc[0]['company_name'] == self._get_independent())):
                LOG.debug('Updating affiliation of user %s to %s',
                          user['user_id'], company_name)
                uc[0]['company_name'] = company_name
                break

    def _get_user_exact_company(self, user):
        if len(user.get('companies', [])) == 1:
            return user['companies'][0]['company_name']
        return None

    def _merge_user_profiles(self, user_profiles):
        LOG.debug('Merge profiles: %s', user_profiles)

        # check of there are more than 1 launchpad_id nor gerrit_id
        lp_ids = set(u.get('launchpad_id') for u in user_profiles
                     if u.get('launchpad_id'))
        if len(lp_ids) > 1:
            LOG.info('Ambiguous launchpad ids: %s on profiles: %s',
                     lp_ids, user_profiles)
        g_ids = set(u.get('gerrit_id') for u in user_profiles
                    if u.get('gerrit_id'))
        if len(g_ids) > 1:
            LOG.info('Ambiguous gerrit ids: %s on profiles: %s',
                     g_ids, user_profiles)

        merged_user = {}  # merged user profile

        # collect ordinary fields
        for key in ['seq', 'user_name', 'user_id', 'gerrit_id', 'github_id',
                    'launchpad_id', 'companies', 'static', 'ldap_id']:
            value = next((v.get(key) for v in user_profiles if v.get(key)),
                         None)
            if value:
                merged_user[key] = value

        # update user_id, prefer it to be equal to launchpad_id
        merged_user['user_id'] = (merged_user.get('launchpad_id') or
{'fraction_non_alphanumeric': 0.09554505085231342, 'fraction_numerical': 0.0032946569259418423, 'mean_word_length': 2.013813943449169, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1203751', 'n_tokens_mistral': 4116, 'n_tokens_neox': 3839, 'n_words': 1065}