<commit_before>#!/usr/bin/env python
# encoding: utf-8

import codecs
from datetime import datetime
from optparse import OptionParser
import random
import re
import time
import cookielib
import urllib
import urllib2
import logging

from bs4 import BeautifulSoup, NavigableString


class Message:
    def __init__(self, thread_url, sender, recipient, timestamp, subject, content, thunderbird=False):
        self.thread_url = thread_url
        self.sender = sender
        self.recipient = recipient
        self.timestamp = timestamp
        self.subject = subject
        self.content = content
        self.thunderbird = thunderbird

    def __str__(self):
        if self.thunderbird:
            msglength = len(self.content)
            subject="OKC Message, length = " + str(msglength).zfill(4)  # leading zeros for message length
            return """
From - %s
From: %s
To: %s
Subject: %s

%s
URL: %s

"""            % (  self.timestamp.strftime('%a %b %d %H:%M:%S %Y') if self.timestamp else None,
                    self.sender,
                    self.recipient,
                    subject,
                    self.content,
                    self.thread_url)
        else:
            return """
URL: %s
From: %s
To: %s
Date: %s
Subject: %s
Content-Length: %d

%s

"""            % (  self.thread_url,
                    self.sender,
                    self.recipient,
                    self.timestamp,
                    self.subject.strip() if self.subject else None,
                    len(self.content),
                    self.content
                    )


class MessageMissing(Message):

    def __init__(self, thread_url):
        self.thread_url = thread_url
        self.sender = None
        self.recipient = None
        self.timestamp = None
        self.subject = None
        self.content = "ERROR: message(s) not fetched"
        self.thunderbird = False


class ArrowFetcher:
    secure_base_url = 'https://www.okcupid.com'
    sleep_duration = 2.0  # base time to wait after each HTTP request, but this will be adjusted randomly
    encoding_pairs = [('<br />', '\n'),
                      ('<br/>', '\n'),
                      ('&#35;', '#'),
                      ('&amp;', '&'),
                      ('&#38;', '&'),
                      ('&#38;amp;', '&'),
                      ('&lt;', '<'),
                      ('&gt;', '>'),
                      ('&quot;', '"'),
                      ('&#38;quot;', '"'),
                      ('&#39;', "'"),
                      ('&rsquo;', u'\u2019'),
                      ('&mdash;', "--")]

    def __init__(self, username, thunderbird=False, debug=False):
        self.username = username
        self.thunderbird = thunderbird
        self.debug = debug
        self.thread_urls = []

    def _safely_soupify(self, f):
        f = f.partition("function autocoreError")[0] + '</body></html>' # wtf okc with the weirdly encoded "</scr' + 'ipt>'"-type statements in your javascript
        return(BeautifulSoup(f, "html.parser"))

    def _request_read_sleep(self, url):
        f = urllib2.urlopen(url).read()
        time.sleep(abs(self.sleep_duration + (random.randrange(-100, 100)/100.0)))
        return f

    def queue_threads(self):
        self.thread_urls = []
        try:
            for folder in range(1, 4):  # Inbox, Sent, Smiles
                page = 0
                while (page < 1 if self.debug else True):
                    logging.info("Queuing folder %s, page %s", folder, page)
                    f = self._request_read_sleep(self.secure_base_url + '/messages?folder=' + str(folder) + '&low=' + str((page * 30) + 1))
                    soup = self._safely_soupify(f)
                    end_pattern = re.compile('&folder=\d\';')
                    threads = [
                        re.sub(end_pattern, '', li.find('a', {'class': 'open'} )['href'].partition('&folder=')[0])
                        for li in soup.find('ul', {'id': 'messages'}).find_all('li')
                    ]
                    if len(threads) == 0:  # break out of the infinite loop when we reach the end and there are no threads on the page
                        break
                    else:
                        self.thread_urls.extend(threads)
                        page = page + 1
        except AttributeError:<commit_msg>Get message thread id (url) from data property<commit_after>#!/usr/bin/env python
# encoding: utf-8

import codecs
from datetime import datetime
from optparse import OptionParser
import random
import re
import time
import cookielib
import urllib
import urllib2
import logging

from bs4 import BeautifulSoup, NavigableString


class Message:
    def __init__(self, thread_url, sender, recipient, timestamp, subject, content, thunderbird=False):
        self.thread_url = thread_url
        self.sender = sender
        self.recipient = recipient
        self.timestamp = timestamp
        self.subject = subject
        self.content = content
        self.thunderbird = thunderbird

    def __str__(self):
        if self.thunderbird:
            msglength = len(self.content)
            subject="OKC Message, length = " + str(msglength).zfill(4)  # leading zeros for message length
            return """
From - %s
From: %s
To: %s
Subject: %s

%s
URL: %s

"""            % (  self.timestamp.strftime('%a %b %d %H:%M:%S %Y') if self.timestamp else None,
                    self.sender,
                    self.recipient,
                    subject,
                    self.content,
                    self.thread_url)
        else:
            return """
URL: %s
From: %s
To: %s
Date: %s
Subject: %s
Content-Length: %d

%s

"""            % (  self.thread_url,
                    self.sender,
                    self.recipient,
                    self.timestamp,
                    self.subject.strip() if self.subject else None,
                    len(self.content),
                    self.content
                    )


class MessageMissing(Message):

    def __init__(self, thread_url):
        self.thread_url = thread_url
        self.sender = None
        self.recipient = None
        self.timestamp = None
        self.subject = None
        self.content = "ERROR: message(s) not fetched"
        self.thunderbird = False


class ArrowFetcher:
    secure_base_url = 'https://www.okcupid.com'
    sleep_duration = 2.0  # base time to wait after each HTTP request, but this will be adjusted randomly
    encoding_pairs = [('<br />', '\n'),
                      ('<br/>', '\n'),
                      ('&#35;', '#'),
                      ('&amp;', '&'),
                      ('&#38;', '&'),
                      ('&#38;amp;', '&'),
                      ('&lt;', '<'),
                      ('&gt;', '>'),
                      ('&quot;', '"'),
                      ('&#38;quot;', '"'),
                      ('&#39;', "'"),
                      ('&rsquo;', u'\u2019'),
                      ('&mdash;', "--")]

    def __init__(self, username, thunderbird=False, debug=False):
        self.username = username
        self.thunderbird = thunderbird
        self.debug = debug
        self.thread_urls = []

    def _safely_soupify(self, f):
        f = f.partition("function autocoreError")[0] + '</body></html>' # wtf okc with the weirdly encoded "</scr' + 'ipt>'"-type statements in your javascript
        return(BeautifulSoup(f, "html.parser"))

    def _request_read_sleep(self, url):
        f = urllib2.urlopen(url).read()
        time.sleep(abs(self.sleep_duration + (random.randrange(-100, 100)/100.0)))
        return f

    def queue_threads(self):
        self.thread_urls = []
        try:
            for folder in range(1, 4):  # Inbox, Sent, Smiles
                page = 0
                while (page < 1 if self.debug else True):
                    logging.info("Queuing folder %s, page %s", folder, page)
                    f = self._request_read_sleep(self.secure_base_url + '/messages?folder=' + str(folder) + '&low=' + str((page * 30) + 1))
                    soup = self._safely_soupify(f)
                    end_pattern = re.compile('&folder=\d\';')
                    threads = [
                        '/messages?readmsg=true&threadid=' + li['data-threadid']
                        for li in soup.find('ul', {'id': 'messages'}).find_all('li')
                    ]
                    if len(threads) == 0:  # break out of the infinite loop when we reach the end and there are no threads on the page
                        break
                    else:
                        self.thread_urls.extend(threads)
                        page = page + 1
        except AttributeError:
{'fraction_non_alphanumeric': 0.12852700679488657, 'fraction_numerical': 0.009558908211447656, 'mean_word_length': 1.6156626506024097, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 15, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3584359', 'n_tokens_mistral': 2557, 'n_tokens_neox': 2388, 'n_words': 621}