<commit_before># This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.

import datetime
import re

from pymongo import MongoClient
import pymongo.errors

__document_version__ = "0.1"

class ContactFilter(object):
    def __init__(self, parent = None):
        self.parent = parent
        return

    def process(self, message):
        phones = re.findall("\d{3}[-/]\d{3}[-/]\d{4}|\d{10}", message.source)
        emails = re.findall("[\w._-]+\@[\w_-]+\.\w+", message.source)
        today = datetime.datetime.now()
        res = {
                "_id": message.url,
                "v": __document_version__,
                "source": message.source,
                "contact": {
                    "emails": list(set(emails)),
                    "phones": list(set(phones))
                    },
                "dateRange": {
                    "first": today,
                    "last": today,
                }
                }
        if self.parent:
            res = self.parent.process(message)
        contact = {}
        contact["emails"] = emails
        contact["phones"] = phones
        res["contact"] = contact
        return res

class BackPageUrlParser(object):
    def __init__(self, parent = None):
        self.parent = parent
        return

    def process(self, message):
        parsed = re.findall("http://(\w+)\.(\w+)\.com/", message.url)
        today = datetime.datetime.now()<commit_msg>Improve phone and email regex<commit_after># This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.

import datetime
import re

from pymongo import MongoClient
import pymongo.errors

__document_version__ = "0.1"

class ContactFilter(object):
    def __init__(self, parent = None):
        self.parent = parent
        return

    def process(self, message):
        # contact info may be in the email or the url
        phones = re.findall("(\d{3})\D*(\d{3})\D*(\d{4})", message.source)
        phones.extend(re.findall("(\d{3})\D*(\d{3})\D*(\d{4})", message.url))
        emails = re.findall("[\w._-]+\@[\w_-]+\.\w+", message.source)
        emails.extend(re.findall("[\w._-]+\@[\w_-]+\.\w+", message.url))
        today = datetime.datetime.now()
        res = {
                "_id": message.url,
                "v": __document_version__,
                "source": message.source,
                "contact": {
                    # store only unique emails and phones
                    "emails": list(set(emails)),
                    "phones": list(set(["-".join(x) for x in phones]))
                    },
                "dateRange": {
                    "first": today,
                    "last": today,
                }
                }
        if self.parent:
            res = self.parent.process(message)
        contact = {}
        contact["emails"] = emails
        contact["phones"] = phones
        res["contact"] = contact
        return res

class BackPageUrlParser(object):
    def __init__(self, parent = None):
        self.parent = parent
        return

    def process(self, message):
        parsed = re.findall("http://(\w+)\.(\w+)\.com/", message.url)
        today = datetime.datetime.now()
{'fraction_non_alphanumeric': 0.14489380273494326, 'fraction_numerical': 0.00669188245562991, 'mean_word_length': 1.7905844155844155, 'pattern_counts': {'":': 18, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3479841', 'n_tokens_mistral': 1022, 'n_tokens_neox': 983, 'n_words': 255}