<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

from __future__ import (division, print_function, absolute_import,
                        unicode_literals)
from crawler.repos import get_random_repos, process_repo

ntot = 0
nreadme = 0
nlicense = 0
while True:
    # Get a list of random repos.
    try:
        repos = get_random_repos()
    except Exception as e:
        print("While getting repos, got exception: ")
        print(e)
        continue

    # Loop over these repos and process each one.
    r = []
    for repo in repos:
        try:
            r.append(process_repo(repo))
        except Exception as e:
            print("While analyzing a repo, got exception: ")
            print(e)
            continue

    # Count the number of readmes and licenses.
    r = map(sum, zip(*r))
    nreadme += r[0]
    nlicense += r[1]
    ntot += len(repos)

    # Watch the grass grow.
    print("Analyzed {0} repositories. Found: {1} readmes, {2} licenses"
          .format(ntot, nreadme, nlicense))<commit_msg>more verbose on the errors<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

from __future__ import (division, print_function, absolute_import,
                        unicode_literals)
import time
import traceback
from crawler.repos import get_random_repos, process_repo

ntot = 0
nreadme = 0
nlicense = 0
while True:
    # Get a list of random repos.
    strt = time.time()
    try:
        repos = get_random_repos()
    except Exception as e:
        print("While getting repos, got exception: ")
        traceback.print_exc()
        continue

    # Loop over these repos and process each one.
    r = []
    for repo in repos:
        try:
            r.append(process_repo(repo))
        except Exception as e:
            print("While analyzing a repo, got exception: ")
            traceback.print_exc()
            continue

    # Count the number of readmes and licenses.
    r = map(sum, zip(*r))
    nreadme += r[0]
    nlicense += r[1]
    ntot += len(repos)

    # Watch the grass grow.
    print(("Analyzed {0} repositories. Found: {1} readmes, {2} licenses. "
           "Took {0} seconds.").format(ntot, nreadme, nlicense,
                                       time.time() - strt))
{'fraction_non_alphanumeric': 0.10195903829029386, 'fraction_numerical': 0.008459483526268923, 'mean_word_length': 2.1426573426573428, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5535661', 'n_tokens_mistral': 710, 'n_tokens_neox': 660, 'n_words': 217}