<commit_before>#!/usr/bin/env python

# License: none (public domain)

import gzip
import hashlib
import json
import logging
import pickle
import re
import smtplib
import string
import urllib2


from datetime import datetime, time
from email.mime.multipart import MIMEMultipart
from email.mime.text import MIMEText
from itertools import groupby
from pytz import timezone


# setup logging to pancake.log and console
log = logging.getLogger('pancake')
log.setLevel(logging.DEBUG)

fh = logging.FileHandler('pancake.log')
fh.setLevel(logging.DEBUG)

ch = logging.StreamHandler()
ch.setLevel(logging.DEBUG)

formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
ch.setFormatter(formatter)
fh.setFormatter(formatter)

log.addHandler(ch)<commit_msg>Note about pytz being third party.<commit_after>#!/usr/bin/env python

# License: none (public domain)

import gzip
import hashlib
import json
import logging
import pickle
import re
import smtplib
import string
import urllib2


from datetime import datetime, time
from email.mime.multipart import MIMEMultipart
from email.mime.text import MIMEText
from itertools import groupby
from pytz import timezone # third party; is there a standard solution?


# setup logging to pancake.log and console
log = logging.getLogger('pancake')
log.setLevel(logging.DEBUG)

fh = logging.FileHandler('pancake.log')
fh.setLevel(logging.DEBUG)

ch = logging.StreamHandler()
ch.setLevel(logging.DEBUG)

formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
ch.setFormatter(formatter)
fh.setFormatter(formatter)

log.addHandler(ch)
{'fraction_non_alphanumeric': 0.09888751545117429, 'fraction_numerical': 0.0012360939431396785, 'mean_word_length': 5.831223628691983, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6380030', 'n_tokens_mistral': 536, 'n_tokens_neox': 516, 'n_words': 147}