<commit_before>from datetime import datetime, time
from hashlib import md5

from cwpoliticl import settings
from wordpress_xmlrpc import Client, WordPressPost, xmlrpc_client
from wordpress_xmlrpc.methods.posts import GetPosts, NewPost
from wordpress_xmlrpc.methods.users import GetUserInfo
from BeautifulSoup import BeautifulSoup
from wordpress_xmlrpc import Client, WordPressPost
from wordpress_xmlrpc.methods import posts, media, taxonomies
from wordpress_xmlrpc.compat import xmlrpc_client
from wordpress_xmlrpc.methods.posts import GetPosts, NewPost
from wordpress_xmlrpc.methods.users import GetUserInfo
from wordpress_xmlrpc.methods import taxonomies

from cwpoliticl.extensions.rpc.wordpress_xml_rpc_utils import WDXmlRPCUtils
from cwpoliticl.utils.images_downloader import ImagesDownload
import os
import urllib2

USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_5) AppleWebKit/537.36 '' \
''(KHTML, like Gecko) Chrome/42.0.2311.135 Safari/537.36'


class TheViewsPaperXmlRPCUtils(WDXmlRPCUtils):
    def __init__(self, wd_host, wd_user, wd_passwd):
        url = "{}/xmlrpc.php".format(wd_host)
        self.wp = Client(url, wd_user, wd_passwd)
        super(TheViewsPaperXmlRPCUtils, self).__init__(wd_host, wd_user, wd_passwd)

    def post_to_wd_for_theviewspaper(self, item, access_denied_cookie):
        image_link = item['image_src']
        # step 1: Download the featured image to the template folder.
        image_location = ImagesDownload.get_image_location(image_link)
        if not os.path.exists(image_location):
            self.download_image(image_link, image_location, access_denied_cookie)

        image_location = image_location

    def download_image(image_link, path, referer=None, cookie=None):
        req = urllib2.Request(image_link)
        if referer:
            req.add_header("Referer", referer)
        if cookie:
            req.add_header("Cookie", cookie)
        req.add_header("User-Agent", USER_AGENT)
        u = urllib2.urlopen(req)
        f = open(path, 'wb')
        meta = u.info()
        file_size = int(meta.getheaders("Content-Length")[0])
        # print u"Downloading: %s Bytes: %s" % (path, file_size)

        file_size_dl = 0
        block_sz = 8192
        while True:<commit_msg>cwpoliticl.<commit_after>from datetime import datetime, time
from hashlib import md5

from cwpoliticl import settings
from wordpress_xmlrpc import Client, WordPressPost, xmlrpc_client
from wordpress_xmlrpc.methods.posts import GetPosts, NewPost
from wordpress_xmlrpc.methods.users import GetUserInfo
from BeautifulSoup import BeautifulSoup
from wordpress_xmlrpc import Client, WordPressPost
from wordpress_xmlrpc.methods import posts, media, taxonomies
from wordpress_xmlrpc.compat import xmlrpc_client
from wordpress_xmlrpc.methods.posts import GetPosts, NewPost
from wordpress_xmlrpc.methods.users import GetUserInfo
from wordpress_xmlrpc.methods import taxonomies

from cwpoliticl.extensions.rpc.wordpress_xml_rpc_utils import WDXmlRPCUtils
from cwpoliticl.utils.images_downloader import ImagesDownload
import os
import urllib2

USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_5) AppleWebKit/537.36 '' \
''(KHTML, like Gecko) Chrome/42.0.2311.135 Safari/537.36'


class TheViewsPaperXmlRPCUtils(WDXmlRPCUtils):
    def __init__(self, wd_host, wd_user, wd_passwd):
        super(TheViewsPaperXmlRPCUtils, self).__init__(wd_host, wd_user, wd_passwd)

    def post_to_wd_for_theviewspaper(self, item, access_denied_cookie):
        image_link = item['image_src']
        # step 1: Download the featured image to the template folder.
        image_location = ImagesDownload.get_image_location(image_link)
        if not os.path.exists(image_location):
            self.download_image(image_link, image_location, access_denied_cookie)

        image_location = image_location

    def download_image(image_link, path, referer=None, cookie=None):
        req = urllib2.Request(image_link)
        if referer:
            req.add_header("Referer", referer)
        if cookie:
            req.add_header("Cookie", cookie)
        req.add_header("User-Agent", USER_AGENT)
        u = urllib2.urlopen(req)
        f = open(path, 'wb')
        meta = u.info()
        file_size = int(meta.getheaders("Content-Length")[0])
        # print u"Downloading: %s Bytes: %s" % (path, file_size)

        file_size_dl = 0
        block_sz = 8192
        while True:
{'fraction_non_alphanumeric': 0.07997281377435432, 'fraction_numerical': 0.01676483914816493, 'mean_word_length': 4.074712643678161, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 25}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 14, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6890434', 'n_tokens_mistral': 1448, 'n_tokens_neox': 1390, 'n_words': 329}