<commit_before># vim: set columns=80:
import os
import datetime
import csv
import urllib2

import simplejson
import datetime
import glob
import lxml
from ..search.models import Project, Bug
import codecs

def get_tag_text_from_xml(xml_doc, tag_name, index = 0):
    """Given an object representing <bug><tag>text</tag></bug>,
    and tag_name = 'tag', returns 'text'."""
    tags = xml_doc.xpath(tag_name)
    try:
        return tags[index].text
    except IndexError:
        return ''
    assert False, "You should not get here."

def count_people_involved(xml_doc):
    """Strategy: Create a set of all the listed text values
    inside a <who ...>(text)</who> tag
    Return the length of said set."""
    everyone = [tag.text for tag in xml_doc.xpath('who')]
    return len(set(everyone))

def bugzilla_date_to_datetime(date_string):
    # FIXME: I make guesses as to the timezone.
    try:
        ret = datetime.datetime.strptime(date_string, '%Y-%m-%d %H:%M')
    except ValueError:
        try:
            ret = datetime.datetime.strptime(date_string, '%Y-%m-%d %H:%M:%S')
        except ValueError:
            ret = datetime.datetime.strptime(date_string, '%Y-%m-%d %H:%M:%S +0000') # UTC-only, baby
    return ret

def who_tag_to_username_and_realname(who_tag):
    username = who_tag.text
    realname = who_tag.attrib['name']
    return username, realname

def xml2bug_object(xml_fd,
                   canonical_bug_link_format_string=
                   'http://bugzilla.pculture.org/show_bug.cgi?id=%d'):
    """xml fd: xml file descriptor 'containing' information about one bug"""
    parsed = lxml.etree.parse(xml_fd)
    gen_miro_project = lambda x: Project.objects.get_or_create(name='Miro')[0]
    return bug_elt2bug_object(bug_elt, canonical_bug_link_format_string, gen_miro_project)

def bug_elt2bug_object(parsed, canonical_bug_link_format_string,
                       gen_project):
    date_reported_text = get_tag_text_from_xml(parsed, 'creation_ts')
    last_touched_text = get_tag_text_from_xml(parsed, 'delta_ts')
    u, r = who_tag_to_username_and_realname(parsed.xpath('.//who')[0])
    bug_id = int(get_tag_text_from_xml(parsed, 'bug_id'))
    keywords_text = get_tag_text_from_xml(parsed, 'keywords')
    keywords = map(lambda s: s.strip(),
                   keywords_text.split(','))
    project = gen_project(parsed)
    status = get_tag_text_from_xml(parsed, 'bug_status')
    looks_closed = status in ('RESOLVED', 'WONTFIX')

    ret = Bug(
        project = project,
        title = get_tag_text_from_xml(parsed, 'short_desc'),
        description = (get_tag_text_from_xml(parsed, 'long_desc/thetext') or
                       get_tag_text_from_xml(parsed, 'short_desc')),
        status = status,
        importance = get_tag_text_from_xml(parsed, 'bug_severity'),
        people_involved = count_people_involved(parsed),
        date_reported = bugzilla_date_to_datetime(date_reported_text),
        last_touched = bugzilla_date_to_datetime(last_touched_text),
        last_polled = datetime.datetime.now(),
        submitter_username = u,
        submitter_realname = r,
        canonical_bug_link = canonical_bug_link_format_string % bug_id,
        good_for_newcomers = ('bitesized' in keywords),
        looks_closed=looks_closed)
    return ret

def bugzilla_query_to_bug_ids(csv_fd):
    doc = csv.reader(csv_fd)
    try:
        doc.next() # throw away header row
    except StopIteration:
        return []

    bug_ids = []
    
    for row in doc:
        bug_ids.append(int(row[0]))

    return bug_ids

def link2bug_id(url):<commit_msg>Empty description marked that way<commit_after># vim: set columns=80:
import os
import datetime
import csv
import urllib2

import simplejson
import datetime
import glob
import lxml
from ..search.models import Project, Bug
import codecs

def get_tag_text_from_xml(xml_doc, tag_name, index = 0):
    """Given an object representing <bug><tag>text</tag></bug>,
    and tag_name = 'tag', returns 'text'."""
    tags = xml_doc.xpath(tag_name)
    try:
        return tags[index].text
    except IndexError:
        return ''
    assert False, "You should not get here."

def count_people_involved(xml_doc):
    """Strategy: Create a set of all the listed text values
    inside a <who ...>(text)</who> tag
    Return the length of said set."""
    everyone = [tag.text for tag in xml_doc.xpath('who')]
    return len(set(everyone))

def bugzilla_date_to_datetime(date_string):
    # FIXME: I make guesses as to the timezone.
    try:
        ret = datetime.datetime.strptime(date_string, '%Y-%m-%d %H:%M')
    except ValueError:
        try:
            ret = datetime.datetime.strptime(date_string, '%Y-%m-%d %H:%M:%S')
        except ValueError:
            ret = datetime.datetime.strptime(date_string, '%Y-%m-%d %H:%M:%S +0000') # UTC-only, baby
    return ret

def who_tag_to_username_and_realname(who_tag):
    username = who_tag.text
    realname = who_tag.attrib['name']
    return username, realname

def xml2bug_object(xml_fd,
                   canonical_bug_link_format_string=
                   'http://bugzilla.pculture.org/show_bug.cgi?id=%d'):
    """xml fd: xml file descriptor 'containing' information about one bug"""
    parsed = lxml.etree.parse(xml_fd)
    gen_miro_project = lambda x: Project.objects.get_or_create(name='Miro')[0]
    return bug_elt2bug_object(bug_elt, canonical_bug_link_format_string, gen_miro_project)

def bug_elt2bug_object(parsed, canonical_bug_link_format_string,
                       gen_project):
    date_reported_text = get_tag_text_from_xml(parsed, 'creation_ts')
    last_touched_text = get_tag_text_from_xml(parsed, 'delta_ts')
    u, r = who_tag_to_username_and_realname(parsed.xpath('.//who')[0])
    bug_id = int(get_tag_text_from_xml(parsed, 'bug_id'))
    keywords_text = get_tag_text_from_xml(parsed, 'keywords')
    keywords = map(lambda s: s.strip(),
                   keywords_text.split(','))
    project = gen_project(parsed)
    status = get_tag_text_from_xml(parsed, 'bug_status')
    looks_closed = status in ('RESOLVED', 'WONTFIX')

    ret = Bug(
        project = project,
        title = get_tag_text_from_xml(parsed, 'short_desc'),
        description = (get_tag_text_from_xml(parsed, 'long_desc/thetext') or
                       '(Empty description)'),
        status = status,
        importance = get_tag_text_from_xml(parsed, 'bug_severity'),
        people_involved = count_people_involved(parsed),
        date_reported = bugzilla_date_to_datetime(date_reported_text),
        last_touched = bugzilla_date_to_datetime(last_touched_text),
        last_polled = datetime.datetime.now(),
        submitter_username = u,
        submitter_realname = r,
        canonical_bug_link = canonical_bug_link_format_string % bug_id,
        good_for_newcomers = ('bitesized' in keywords),
        looks_closed=looks_closed)
    return ret

def bugzilla_query_to_bug_ids(csv_fd):
    doc = csv.reader(csv_fd)
    try:
        doc.next() # throw away header row
    except StopIteration:
        return []

    bug_ids = []
    
    for row in doc:
        bug_ids.append(int(row[0]))

    return bug_ids

def link2bug_id(url):
{'fraction_non_alphanumeric': 0.1075403450194769, 'fraction_numerical': 0.004173622704507512, 'mean_word_length': 3.354330708661417, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 41}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5287763', 'n_tokens_mistral': 2409, 'n_tokens_neox': 2380, 'n_words': 505}