<commit_before>import itertools
import sys
import random
import logging
from collections import defaultdict
from multiprocessing import Pool

from settings import settings
from base.models import *
from base.utils import use_mongo, write_json

def save_mod_group():
    for u in User.find(User.median_loc.exists()):
        u.mod_group = u._id%100
        u.save()

def prep_eval_users(key='50'):
    use_mongo('usgeo')
    logging.info('started %s',key)
    users = User.find(User.mod_group == int(key))
    write_json(itertools.imap(_edges_d, users), "data/eval"+key)
    logging.info('saved %s',key)


def _edges_d(me):
    #./do.py save_user_json edges_json 
    tweets = Tweets.get_id(me._id,fields=['ats'])
    edges = Edges.get_id(me._id)
    ats = set(tweets.ats or [])
    frds = set(edges.friends or [])
    fols = set(edges.followers or [])
    
    group_order = (ats,frds,fols)
    def _group(uid):
        #return 7 for ated rfrd, 4 for ignored jfol
        return sum(2**i for i,s in enumerate(group_order) if uid in s)

    #pick the 100 best users
    lookups = edges.lookups if edges.lookups else list(ats|frds|fols)
    random.shuffle(lookups)
    lookups.sort(key=_group, reverse=True)
    lookups = lookups[:100]

    #get the users - this will be SLOW
    amigos = User.find(
            User._id.is_in(lookups) & User.geonames_place.exists(),
            fields =['gnp','folc'],
            )

    rels = [_rel_d(amigo,_group(amigo._id)) for amigo in amigos]
    return dict(
            _id = me._id,
            mloc = me.median_loc,
            rels = rels,
            lu_len = len(lookups)
            )

<commit_msg>added protected field to prep<commit_after>import itertools
import sys
import random
import logging
from collections import defaultdict
from multiprocessing import Pool

from settings import settings
from base.models import *
from base.utils import use_mongo, write_json

def save_mod_group():
    for u in User.find(User.median_loc.exists()):
        u.mod_group = u._id%100
        u.save()

def prep_eval_users(key='50'):
    use_mongo('usgeo')
    logging.info('started %s',key)
    users = User.find(User.mod_group == int(key))
    write_json(itertools.imap(_edges_d, users), "data/eval"+key)
    logging.info('saved %s',key)


def _edges_d(me):
    #./do.py save_user_json edges_json 
    tweets = Tweets.get_id(me._id,fields=['ats'])
    edges = Edges.get_id(me._id)
    ats = set(tweets.ats or [])
    frds = set(edges.friends or [])
    fols = set(edges.followers or [])
    
    group_order = (ats,frds,fols)
    def _group(uid):
        #return 7 for ated rfrd, 4 for ignored jfol
        return sum(2**i for i,s in enumerate(group_order) if uid in s)

    #pick the 100 best users
    lookups = edges.lookups if edges.lookups else list(ats|frds|fols)
    random.shuffle(lookups)
    lookups.sort(key=_group, reverse=True)
    lookups = lookups[:100]

    #get the users - this will be SLOW
    amigos = User.find(
            User._id.is_in(lookups) & User.geonames_place.exists(),
            fields =['gnp','folc'],
            )

    res = dict(
        _id = me._id,
        mloc = me.median_loc,
        lu_len = len(lookups),
        rels = [
            _rel_d(amigo, _group(amigo._id) + (8 if amigo.protected else 0))
            for amigo in amigos],
        )
    if me.geonames_place:
        res['gnp'] = me.geonames_place.to_d()
    return res


{'fraction_non_alphanumeric': 0.11888726207906296, 'fraction_numerical': 0.008784773060029283, 'mean_word_length': 2.7787610619469025, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '58390', 'n_tokens_mistral': 1228, 'n_tokens_neox': 1187, 'n_words': 286}