<commit_before># -*- coding: utf-8 -*-
""" Code to find out about common things in both the crunchbase data
and the Top1M data.

What you should do before starting to use this module:

    * generate key files

    * generate a name count

    * fetch all data from crunchbase
"""

import collections
import glob
import math
import os.path

import crawlcrunch.model.db as ccdb

from midas.compat import GzipFile
from midas.compat import urlparse
from midas.scripts.alexa_to_key_files import check_and_count_entries

import midas.config as md_cfg

_sess = None

def make_session(db=None):
    global _sess
    if _sess is None:
        if db is None:
            db = md_cfg.get('statistics', 'crunchbase_db')
        engine = ccdb.create_engine(db)
        ccdb.Session.configure(bind=engine)
        _sess = ccdb.Session()
    return _sess

def check_sha1_distr_mean_max_min_deviation_variance(root_dir=None):
    if root_dir is None:
        root_dir = md_cfg.get('statistics', 'key_files')
    counter = check_and_count_entries(glob.glob(os.path.join(root_dir, '*')))
    mean = sum(counter.itervalues()) / len(counter) * 1.0
    max_ = max(counter.itervalues())
    min_ = min(counter.itervalues())
    deviation = (sum(math.fabs(x - mean) for x in counter.itervalues())
                 / len(counter))
    variance = deviation**2
    return (mean, max_, min_, deviation, variance)
    

SiteCnt = collections.namedtuple('SiteCount', ['site', 'cnt'])

def get_site_counts(path=None):
    if path is None:
        path = md_cfg.get('statistics', 'site_count')
    with GzipFile(path) as fp:
        for l in fp:
            site, cnt = l.decode().strip().split('\t', 1)
            cnt = int(cnt)
            yield SiteCnt(site, cnt)

def all_companies():
    " Returns 100355 companies. "
    sess = make_session()
    return sess.query(ccdb.Company).all()<commit_msg>define the companies of interest<commit_after># -*- coding: utf-8 -*-
""" Code to find out about common things in both the crunchbase data
and the Top1M data.

What you should do before starting to use this module:

    * generate key files

    * generate a name count

    * fetch all data from crunchbase
"""

import collections
import glob
import math
import os.path

import crawlcrunch.model.db as ccdb

from midas.compat import GzipFile
from midas.compat import urlparse
from midas.scripts.alexa_to_key_files import check_and_count_entries

import midas.config as md_cfg

_sess = None

def make_session(db=None):
    global _sess
    if _sess is None:
        if db is None:
            db = md_cfg.get('statistics', 'crunchbase_db')
        engine = ccdb.create_engine(db)
        ccdb.Session.configure(bind=engine)
        _sess = ccdb.Session()
    return _sess

def check_sha1_distr_mean_max_min_deviation_variance(root_dir=None):
    if root_dir is None:
        root_dir = md_cfg.get('statistics', 'key_files')
    counter = check_and_count_entries(glob.glob(os.path.join(root_dir, '*')))
    mean = sum(counter.itervalues()) / len(counter) * 1.0
    max_ = max(counter.itervalues())
    min_ = min(counter.itervalues())
    deviation = (sum(math.fabs(x - mean) for x in counter.itervalues())
                 / len(counter))
    variance = deviation**2
    return (mean, max_, min_, deviation, variance)
    

SiteCnt = collections.namedtuple('SiteCount', ['site', 'cnt'])

def get_site_counts(path=None):
    if path is None:
        path = md_cfg.get('statistics', 'site_count')
    with GzipFile(path) as fp:
        for l in fp:
            site, cnt = l.decode().strip().split('\t', 1)
            cnt = int(cnt)
            yield SiteCnt(site, cnt)

def all_companies():
    " Returns 100355 companies. "
    sess = make_session()
    return sess.query(ccdb.Company).all()

def companies_of_interest():
    """ Returns all companies with a funding round with round_level
    'angel', 'seed' or 'a' since December 2010.
    """
    sess = make_session()
    q = sess.query(ccdb.Company).join(ccdb.FundingRound)\
        .filter(or_(ccdb.FundingRound.funded_year>2010,
                    and_(ccdb.FundingRound.funded_year==2010,
                         ccdb.FundingRound.funded_month==12)))\
        .filter(ccdb.FundingRound.round_code.in_(['angel', 'seed', 'a']))
    return q.all()
{'fraction_non_alphanumeric': 0.11220196353436185, 'fraction_numerical': 0.009350163627863487, 'mean_word_length': 3.067490494296578, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1159500', 'n_tokens_mistral': 1445, 'n_tokens_neox': 1394, 'n_words': 367}