<commit_before>#coding: utf-8
import random
import urllib
import json
import xml.etree.ElementTree as ET
from ftplib import FTP
from tempfile import NamedTemporaryFile

from datetime import datetime

from django.utils.text import slugify

from .base import BaseProcessor
from .efe import iptc
from .category_efe import CATEGORY_EFE

from opps.articles.models import Post
from opps.channels.models import Channel


class EFEXMLProcessor(BaseProcessor):

    def connect(self):
        self.ftp = FTP()
        params = {"host": self.feed.source_url}
        if self.feed.source_port:
            params['port'] = int(self.feed.source_port)

        self.ftp.connect(**params)

        if self.feed.source_username:
            self.ftp.login(self.feed.source_username,
                           self.feed.source_password)

        self.verbose_print(self.ftp.getwelcome())

        return self.ftp

    def get_temp_file(self):
        f = NamedTemporaryFile(delete=True)
        self.verbose_print("%s tempfile created" % f.name)
        return f

    def process_file(self, s):

        self.verbose_print("-" * 78)

        if self.log_created(s):
            self.verbose_print("%s already processed, skipping." % s)
            return

        s = s.strip()
        s = s.replace("\n", "")
        ext = s.split('.')[-1]
        if ext not in ['XML', 'xml']:
            self.verbose_print("Skipping non xml %s" % s)
            return

        self.verbose_print("Retrieving file %s" % s)

        source_root_folder = self.feed.source_root_folder
        if not source_root_folder.endswith('/'):
            source_root_folder += "/"

        if self.feed.source_username:
            url = "ftp://{0}:{1}@{2}{3}{4}".format(self.feed.source_username,
                                                   self.feed.source_password,
                                                   self.feed.source_url,
                                                   source_root_folder,
                                                   s)
        else:
            url = "ftp://{0}{1}{2}".format(self.feed.source_url,
                                       source_root_folder,
                                       s)
        self.verbose_print(url)

        f = self.get_temp_file()
        try:
            urllib.urlretrieve(url, filename=f.name)
            self.verbose_print("File retrieved successfully")
        except Exception as e:
            self.verbose_print("error urlretrieve")
            self.verbose_print(str(e))
            return

        try:
            xml_string = f.read()
            self.verbose_print("xml_string read!")
        except Exception as e:
            self.verbose_print("error f.read")
            self.verbose_print(str(e))
            return

        if not xml_string:
            self.verbose_print("XML Empty")
            f.close()
            return

        data = self.parse_xml(f.name)
        data = self.categorize(data)
        # self.verbose_print(str(data))

        f.close()

        created = self.create_entry(data)
        if created:
            self.record_log(s)
        else:
            self.verbose_print("Entry not created")

    def parse_xml(self, filename):
        data = {}

        try:
            tree = ET.parse(filename)
            root = tree.getroot()
        except:
            return

        # TODO: NewsItem can be multiple items

        try:
            data['headline'] = root.find(
                './NewsItem/NewsComponent/NewsLines/HeadLine').text
            data['subheadline'] = root.find(
                './NewsItem/NewsComponent/NewsLines/SubHeadLine').text
        except:
            pass

        try:
            tobject_attrib = root.find(
                './NewsItem/NewsComponent/ContentItem/'
                'DataContent/nitf/head/tobject/tobject.subject')
            data['iptc_code'] = tobject_attrib.get('tobject.subject.refnum')
            data['iptc_matter'] = tobject_attrib.get('tobject.subject.matter')
            data['iptc_type'] = tobject_attrib.get('tobject.subject.type')
        except:
            pass

        try:
            pub_data_attrib = root.find(
                './NewsItem/NewsComponent/ContentItem/'
                'DataContent/nitf/head/pubdata')
            data['pub_date'] = pub_data_attrib.get('date.publication')
            data['item_len'] = pub_data_attrib.get('item-length')
        except:
            pass

        try:
            data['abstract'] = root.find(
                './NewsItem/NewsComponent/ContentItem'
                '/DataContent/nitf/body/body.head/abstract/').text
        except:
            pass

        try:
            data['owner'] = root.find(
                './NewsItem/NewsComponent/ContentItem/DataContent/nitf/'
                'body/body.head/rights/').text
        except:
            pass

        try:
            data['story_data'] = root.find(
                './NewsItem/NewsComponent/ContentItem/DataContent/nitf/'
                'body/body.head/dateline/story.date').get('norm')
        except:
            pass

        try:
            body = root.find(
                './NewsItem/NewsComponent/ContentItem/DataContent/nitf/'
                'body/body.content')
            data['body'] = u"\n".join(
                u"<p>{0}</p>".format(p.text) for p in body)
        except:
            pass

        if not all([data.get('body'), data.get('headline')]):
            self.verbose_print(
                "Data does not have body and headline %s" % str(data))
            return

        return data

    def parse_dt(self, s):
        self.verbose_print("REceived to parse_dt %s" % s)
        try:
            new_s = datetime.strptime(s[:8], "%Y%m%d")
            self.verbose_print("parsed to %s" % new_s)
            return new_s
        except Exception as e:
            self.verbose_print("CAnt parse dt")
            self.verbose_print(str(e))
            return

    def create_entry(self, data):<commit_msg>Using dateutil trying to parse news date<commit_after>#coding: utf-8
import random
import urllib
import json
import xml.etree.ElementTree as ET
from ftplib import FTP
from tempfile import NamedTemporaryFile

from datetime import datetime
from dateutil.parser import parse

from django.utils.text import slugify

from .base import BaseProcessor
from .efe import iptc
from .category_efe import CATEGORY_EFE

from opps.articles.models import Post
from opps.channels.models import Channel


class EFEXMLProcessor(BaseProcessor):

    def connect(self):
        self.ftp = FTP()
        params = {"host": self.feed.source_url}
        if self.feed.source_port:
            params['port'] = int(self.feed.source_port)

        self.ftp.connect(**params)

        if self.feed.source_username:
            self.ftp.login(self.feed.source_username,
                           self.feed.source_password)

        self.verbose_print(self.ftp.getwelcome())

        return self.ftp

    def get_temp_file(self):
        f = NamedTemporaryFile(delete=True)
        self.verbose_print("%s tempfile created" % f.name)
        return f

    def process_file(self, s):

        self.verbose_print("-" * 78)

        if self.log_created(s):
            self.verbose_print("%s already processed, skipping." % s)
            return

        s = s.strip()
        s = s.replace("\n", "")
        ext = s.split('.')[-1]
        if ext not in ['XML', 'xml']:
            self.verbose_print("Skipping non xml %s" % s)
            return

        self.verbose_print("Retrieving file %s" % s)

        source_root_folder = self.feed.source_root_folder
        if not source_root_folder.endswith('/'):
            source_root_folder += "/"

        if self.feed.source_username:
            url = "ftp://{0}:{1}@{2}{3}{4}".format(self.feed.source_username,
                                                   self.feed.source_password,
                                                   self.feed.source_url,
                                                   source_root_folder,
                                                   s)
        else:
            url = "ftp://{0}{1}{2}".format(self.feed.source_url,
                                       source_root_folder,
                                       s)
        self.verbose_print(url)

        f = self.get_temp_file()
        try:
            urllib.urlretrieve(url, filename=f.name)
            self.verbose_print("File retrieved successfully")
        except Exception as e:
            self.verbose_print("error urlretrieve")
            self.verbose_print(str(e))
            return

        try:
            xml_string = f.read()
            self.verbose_print("xml_string read!")
        except Exception as e:
            self.verbose_print("error f.read")
            self.verbose_print(str(e))
            return

        if not xml_string:
            self.verbose_print("XML Empty")
            f.close()
            return

        data = self.parse_xml(f.name)
        data = self.categorize(data)
        # self.verbose_print(str(data))

        f.close()

        created = self.create_entry(data)
        if created:
            self.record_log(s)
        else:
            self.verbose_print("Entry not created")

    def parse_xml(self, filename):
        data = {}

        try:
            tree = ET.parse(filename)
            root = tree.getroot()
        except:
            return

        # TODO: NewsItem can be multiple items

        try:
            data['headline'] = root.find(
                './NewsItem/NewsComponent/NewsLines/HeadLine').text
            data['subheadline'] = root.find(
                './NewsItem/NewsComponent/NewsLines/SubHeadLine').text
        except:
            pass

        try:
            tobject_attrib = root.find(
                './NewsItem/NewsComponent/ContentItem/'
                'DataContent/nitf/head/tobject/tobject.subject')
            data['iptc_code'] = tobject_attrib.get('tobject.subject.refnum')
            data['iptc_matter'] = tobject_attrib.get('tobject.subject.matter')
            data['iptc_type'] = tobject_attrib.get('tobject.subject.type')
        except:
            pass

        try:
            pub_data_attrib = root.find(
                './NewsItem/NewsComponent/ContentItem/'
                'DataContent/nitf/head/pubdata')
            data['pub_date'] = pub_data_attrib.get('date.publication')
            data['item_len'] = pub_data_attrib.get('item-length')
        except:
            pass

        try:
            data['abstract'] = root.find(
                './NewsItem/NewsComponent/ContentItem'
                '/DataContent/nitf/body/body.head/abstract/').text
        except:
            pass

        try:
            data['owner'] = root.find(
                './NewsItem/NewsComponent/ContentItem/DataContent/nitf/'
                'body/body.head/rights/').text
        except:
            pass

        try:
            data['story_data'] = root.find(
                './NewsItem/NewsComponent/ContentItem/DataContent/nitf/'
                'body/body.head/dateline/story.date').get('norm')
        except:
            pass

        try:
            body = root.find(
                './NewsItem/NewsComponent/ContentItem/DataContent/nitf/'
                'body/body.content')
            data['body'] = u"\n".join(
                u"<p>{0}</p>".format(p.text) for p in body)
        except:
            pass

        if not all([data.get('body'), data.get('headline')]):
            self.verbose_print(
                "Data does not have body and headline %s" % str(data))
            return

        return data

    def parse_dt(self, s):
        self.verbose_print("REceived to parse_dt %s" % s)
        try:
            try:
                new_s = parse(s)
            except:
                new_s = datetime.strptime(s[:8], "%Y%m%d")

            self.verbose_print("parsed to %s" % new_s)
            return new_s
        except Exception as e:
            self.verbose_print("CAnt parse dt")
            self.verbose_print(str(e))
            return

    def create_entry(self, data):
{'fraction_non_alphanumeric': 0.11034369347146851, 'fraction_numerical': 0.002302252918927808, 'mean_word_length': 1.5758153324862347, 'pattern_counts': {'":': 2, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 16}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 20, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1766398', 'n_tokens_mistral': 3499, 'n_tokens_neox': 3365, 'n_words': 647}