<commit_before># -*- encoding: utf-8 -*-
#
# Copyright 2012, <NAME> <<EMAIL>>. All rights reserved.
# License: BSD Style, 2 clauses. see isso/__init__.py
#
# TODO
#
# - export does not include website from commenters
# - Disqus includes already deleted comments

from __future__ import division

import sys
import os

from time import mktime, strptime
from urlparse import urlparse
from collections import defaultdict

from xml.etree import ElementTree

from isso.models import Comment


ns = '{http://disqus.com}'
dsq = '{http://disqus.com/disqus-internals}'


def insert(db, thread, comments):

    path = urlparse(thread.find('%sid' % ns).text).path
    remap = dict()

    for item in sorted(comments, key=lambda k: k['created']):

        parent = remap.get(item.get('dsq:parent'))
        comment = Comment(created=item['created'], text=item['text'],
                          author=item['author'], email=item['email'], parent=parent)

        rv = db.add(path, comment)
        remap[item['dsq:id']] = rv.id


def disqus(db, xmlfile):

    tree = ElementTree.parse(xmlfile)
    res = defaultdict(list)

    for post in tree.findall('%spost' % ns):

        item = {
            'dsq:id': post.attrib.get(dsq + 'id'),
            'text': post.find('%smessage' % ns).text,
            'author': post.find('%sauthor/%sname' % (ns, ns)).text,
            'email': post.find('%sauthor/%semail' % (ns, ns)).text,
            'created': mktime(strptime(
                post.find('%screatedAt' % ns).text, '%Y-%m-%dT%H:%M:%SZ'))
        }

        if post.find(ns + 'parent') is not None:
            item['dsq:parent'] = post.find(ns + 'parent').attrib.get(dsq + 'id')<commit_msg>fix disqus import<commit_after># -*- encoding: utf-8 -*-
#
# Copyright 2012, <NAME> <<EMAIL>>. All rights reserved.
# License: BSD Style, 2 clauses. see isso/__init__.py
#
# TODO
#
# - export does not include website from commenters
# - Disqus includes already deleted comments

from __future__ import division

import sys
import os
import hashlib

from time import mktime, strptime
from urlparse import urlparse
from collections import defaultdict

from xml.etree import ElementTree

from isso.models import Comment


ns = '{http://disqus.com}'
dsq = '{http://disqus.com/disqus-internals}'


def insert(db, thread, comments):

    path = urlparse(thread.find('%sid' % ns).text).path
    remap = dict()

    for item in sorted(comments, key=lambda k: k['created']):

        parent = remap.get(item.get('dsq:parent'))
        comment = Comment(created=item['created'], text=item['text'],
                          author=item['author'], parent=parent,
                          hash=hashlib.md5(item["email"] or '').hexdigest())

        rv = db.add(path, comment, '127.0.0.1')
        remap[item['dsq:id']] = rv["id"]


def disqus(db, xmlfile):

    tree = ElementTree.parse(xmlfile)
    res = defaultdict(list)

    for post in tree.findall('%spost' % ns):

        item = {
            'dsq:id': post.attrib.get(dsq + 'id'),
            'text': post.find('%smessage' % ns).text,
            'author': post.find('%sauthor/%sname' % (ns, ns)).text,
            'email': post.find('%sauthor/%semail' % (ns, ns)).text,
            'created': mktime(strptime(
                post.find('%screatedAt' % ns).text, '%Y-%m-%dT%H:%M:%SZ'))
        }

        if post.find(ns + 'parent') is not None:
            item['dsq:parent'] = post.find(ns + 'parent').attrib.get(dsq + 'id')
{'fraction_non_alphanumeric': 0.16410998552821998, 'fraction_numerical': 0.005499276410998553, 'mean_word_length': 2.84, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 6}, 'pii_count': 1, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1625685', 'n_tokens_mistral': 1167, 'n_tokens_neox': 1119, 'n_words': 274}