<commit_before>import os
import sys
import glob
import datetime
import sqlite3 as lite
from lxml import etree

def yield_all_streams():
    date = lambda x: datetime.datetime.strptime(x, '%c')
    bool_ = lambda x: x != 'False'
    str_ = lambda x: str(x.encode('utf-8'))
    all_attrs = ((bool_, ('abuse_reported', 'channel_subscription',
                    'embed_enabled', 'featured',)),
                (int, ('video_height', 'video_width', 'broadcast_part',
                    'channel_count', 'channel_view_count', 'embed_count', 'id',
                    'site_count', 'stream_count',)),
                (str_, ('audio_codec', 'title', 'broadcaster', 'category',
                    'channel', 'format', 'geo', 'language', 'meta_game', 'name',
                    'stream_type', 'subcategory', 'video_codec',)),
                (float, ('video_bitrate',)),
                (date, ('up_time',)))
    for fname in glob.glob('data/*.xml'):
        try:
            root = etree.parse(fname)
        except etree.XMLSyntaxError as e:
            print e
            continue
        for stream_node in root.xpath('//stream'):
            stream = {}
            for node in stream_node:
                if node.text is None:
                    continue
                data = None
                for type_, attrs in all_attrs:
                    if node.tag in attrs:
                        data = type_(node.text)
                if data is None:
                    continue
                stream[node.tag] = data
            yield stream

conn = lite.connect('twitch.db')
c = conn.cursor()
try:
    c.execute('drop table streams')
except lite.Error as e:
    pass
try:
    c.execute('create table streams (abuse_reported boolean, channel_subscription boolean, embed_enabled boolean, featured boolean, video_height int, video_width int, broadcast_part int, channel_count int, channel_view_count int, embed_count int, id int, site_count int, stream_count int, audio_codec text, title text, broadcaster text, category text, channel text, format text, geo text, language text, meta_game text, name text, stream_type text, subcategory text, video_codec text, video_bitrate real, up_time datetime)')
except lite.Error as e:
    pass

progress = 0.
for stream in yield_all_streams():<commit_msg>primarykey<commit_after>import os
import sys
import glob
import datetime
import sqlite3 as lite
from lxml import etree

def yield_all_streams():
    date = lambda x: datetime.datetime.strptime(x, '%c')
    bool_ = lambda x: x != 'False'
    str_ = lambda x: str(x.encode('utf-8'))
    all_attrs = ((bool_, ('abuse_reported', 'channel_subscription',
                    'embed_enabled', 'featured',)),
                (int, ('video_height', 'video_width', 'broadcast_part',
                    'channel_count', 'channel_view_count', 'embed_count', 'id',
                    'site_count', 'stream_count',)),
                (str_, ('audio_codec', 'title', 'broadcaster', 'category',
                    'channel', 'format', 'geo', 'language', 'meta_game', 'name',
                    'stream_type', 'subcategory', 'video_codec',)),
                (float, ('video_bitrate',)),
                (date, ('up_time',)))
    for fname in glob.glob('data/*.xml'):
        try:
            root = etree.parse(fname)
        except etree.XMLSyntaxError as e:
            print e
            continue
        for stream_node in root.xpath('//stream'):
            stream = {}
            for node in stream_node:
                if node.text is None:
                    continue
                data = None
                for type_, attrs in all_attrs:
                    if node.tag in attrs:
                        data = type_(node.text)
                if data is None:
                    continue
                stream[node.tag] = data
            yield stream

conn = lite.connect('twitch.db')
c = conn.cursor()
try:
    c.execute('drop table streams')
except lite.Error as e:
    pass
try:
    c.execute('create table streams (abuse_reported boolean, channel_subscription boolean, embed_enabled boolean, featured boolean, video_height int, video_width int, broadcast_part int, channel_count int, channel_view_count int, embed_count int, id int not null primary key, site_count int, stream_count int, audio_codec text, title text, broadcaster text, category text, channel text, format text, geo text, language text, meta_game text, name text, stream_type text, subcategory text, video_codec text, video_bitrate real, up_time datetime)')
except lite.Error as e:
    pass

progress = 0.
for stream in yield_all_streams():
{'fraction_non_alphanumeric': 0.10952380952380952, 'fraction_numerical': 0.0012987012987012987, 'mean_word_length': 2.288967971530249, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2909734', 'n_tokens_mistral': 1310, 'n_tokens_neox': 1269, 'n_words': 403}