<commit_before>#!/usr/bin/env python
#
# @author:  <NAME>, <NAME>

import logging
import logging.config
log = logging.getLogger('internims')
logging.getLogger('urllib3').setLevel(logging.WARNING) # silence Requests library logging

import re
import json
import base64
import datetime
import requests
import Crypto.Hash.SHA
import Crypto.PublicKey.RSA
import Crypto.Signature.PKCS1_v1_5


def update(db, api_uri, site_name, site_id, privkey, internims_url):
    """sends is-alive signal to internims central."""
    exp_userlist = [e['permissions'] for e in db.experiments.find(None, {'_id': False, 'permissions.uid': True, 'permissions.site': True})]
    col_userlist = [c['permissions'] for c in db.collections.find(None, {'_id': False, 'permissions.uid': True, 'permissions.site': True})]
    grp_userlist = [g['roles'] for g in db.groups.find(None, {'_id': False, 'roles.uid': True, 'roles.site': True})]
    remote_users = list(set([user['uid']+'#'+user['site'] for container in exp_userlist+col_userlist+grp_userlist for user in container if user.get('site') != None]))

    payload = json.dumps({'site': site_id, 'api_uri': api_uri, 'users': remote_users, 'name': site_name})
    h = Crypto.Hash.SHA.new(payload)
    signature = Crypto.Signature.PKCS1_v1_5.new(privkey).sign(h)
    headers = {'Authorization': base64.b64encode(signature)}

    r = requests.post(internims_url, data=payload, headers=headers)
    if r.status_code == 200:
        response = (json.loads(r.content))
        # update remotes entries
        for site in response['sites']:
            site['timestamp'] = datetime.datetime.strptime(site['timestamp'], '%Y-%m-%dT%H:%M:%S.%fZ')
            db.remotes.update({'_id': site['_id']}, site, upsert=True)
        log.debug('updating remotes: ' + ', '.join((r['_id'] for r in response['sites'])))

        # delete remotes from users, who no longer have remotes
        db.users.update({'remotes': {'$exists': True}, '_id': {'$nin': response['users'].keys()}}, {'$unset': {'remotes': ''}}, multi=True)

        # add remotes to users
        log.debug('users w/ remotes: ' + ', '.join(response['users']))
        for uid, remotes in response['users'].iteritems():
            db.users.update({'_id': uid}, {'$set': {'remotes': remotes}})
        return True
    else:
        # r.reason contains generic description for the specific error code
        # need the part of the error response body that contains the detailed explanation
        reason = re.search('<br /><br />\n(.*)\n\n\n </body>\n</html>', r.content)
        log.warning((r.status_code, reason.group(1)))
        return False


def clean_remotes(db):
    """removes db.remotes, and removes remotes field from all db.users"""<commit_msg>fix logging of internimscentral error responses<commit_after>#!/usr/bin/env python
#
# @author:  <NAME>, <NAME>

import logging
import logging.config
log = logging.getLogger('internims')
logging.getLogger('urllib3').setLevel(logging.WARNING)  # silence Requests library logging

import re
import json
import base64
import datetime
import requests
import Crypto.Hash.SHA
import Crypto.PublicKey.RSA
import Crypto.Signature.PKCS1_v1_5


def update(db, api_uri, site_name, site_id, privkey, internims_url):
    """sends is-alive signal to internims central."""
    exp_userlist = [e['permissions'] for e in db.experiments.find(None, {'_id': False, 'permissions.uid': True, 'permissions.site': True})]
    col_userlist = [c['permissions'] for c in db.collections.find(None, {'_id': False, 'permissions.uid': True, 'permissions.site': True})]
    grp_userlist = [g['roles'] for g in db.groups.find(None, {'_id': False, 'roles.uid': True, 'roles.site': True})]
    remote_users = list(set([user['uid']+'#'+user['site'] for container in exp_userlist+col_userlist+grp_userlist for user in container if user.get('site') is not None]))

    payload = json.dumps({'site': site_id, 'api_uri': api_uri, 'users': remote_users, 'name': site_name})
    h = Crypto.Hash.SHA.new(payload)
    signature = Crypto.Signature.PKCS1_v1_5.new(privkey).sign(h)
    headers = {'Authorization': base64.b64encode(signature)}

    r = requests.post(internims_url, data=payload, headers=headers)
    if r.status_code == 200:
        response = (json.loads(r.content))
        # update remotes entries
        for site in response['sites']:
            site['timestamp'] = datetime.datetime.strptime(site['timestamp'], '%Y-%m-%dT%H:%M:%S.%fZ')
            db.remotes.update({'_id': site['_id']}, site, upsert=True)
        log.debug('updating remotes: ' + ', '.join((r['_id'] for r in response['sites'])))

        # delete remotes from users, who no longer have remotes
        db.users.update({'remotes': {'$exists': True}, '_id': {'$nin': response['users'].keys()}}, {'$unset': {'remotes': ''}}, multi=True)

        # add remotes to users
        log.debug('users w/ remotes: ' + ', '.join(response['users']))
        for uid, remotes in response['users'].iteritems():
            db.users.update({'_id': uid}, {'$set': {'remotes': remotes}})
        return True
    else:
        # r.reason contains generic description for the specific error code
        # need the part of the error response body that contains the detailed explanation
        reason = re.search('<br /><br />\n(.*)\n\n\n </body>\n</html>', r.content)
        if reason:
            msg = reason.group(1)
        else:
            msg = r.reason
        log.warning('%s - %s' % (r.status_code, msg))
        return False


def clean_remotes(db):
    """removes db.remotes, and removes remotes field from all db.users"""
{'fraction_non_alphanumeric': 0.16514582215065307, 'fraction_numerical': 0.006083378064054392, 'mean_word_length': 3.823123382226057, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3903499', 'n_tokens_mistral': 1816, 'n_tokens_neox': 1785, 'n_words': 459}