<commit_before>#!/usr/bin/python
"""
Convert a CSV file into our Beta Whitelist format
"""

try:
  import tldextract
except:
  print("You need to 'pip install -r requirements.txt'")
  sys.exit(1)

import csv, sys, os, re, time, random, shelve
from jinja2 import Template

import malicious_url_check
import argparse
import smtplib

from email.mime.text import MIMEText
from email.MIMEImage import MIMEImage
from email.mime.multipart import MIMEMultipart
from datetime import datetime

# The heading values in the header row for the columns
COL_DOMAIN = "What domain(s) would you like to get a certificate for?"
COL_EMAIL = "What email address can we contact you at when we're ready for you?"

TEMPLATE_DIR = "{0}/templates".format(os.path.dirname(os.path.abspath(__file__)))
DOMAIN_PATTERN = re.compile("^((?!-)[A-Za-z0-9-]{1,63}(?<!-)\.)+[A-Za-z]{2,18}$")
PUNYCODE_PATTERN = re.compile("xn--")

with open(TEMPLATE_DIR+'/btn_donateCC_LG.gif', 'rb') as donateImg:
  DONATE_IMAGE = MIMEImage(donateImg.read())
  DONATE_IMAGE.add_header('Content-ID', '<image1>')

class DomainEntry(object):
  def __init__(self, extracted, domain="", email=""):
    self.extracted = extracted
    self.domain = domain
    self.email = email
    self.problems = []
    self.addedDate = None
    self.safebrowsingDate = None
    self.notificationDate = None

  def __repr__(self):
    msg = "domain={0} email={1}".format(self.domain, self.email)

    if self.problems:
      msg = "{0} problems={1}".format(msg, self.problems)

    if self.safebrowsingDate:
      msg = "{0} S".format(msg)

    if self.addedDate:
      msg = "{0} A".format(msg)

    if self.notificationDate:
      msg = "{0} N".format(msg)

    return msg

  def isInvalid(self):
    if not DOMAIN_PATTERN.match(self.domain):
      return True
    if PUNYCODE_PATTERN.match(self.domain):
      return True
    return False

  def hasProblems(self):
    return len(self.problems) > 0

  def check(self, withMalCheck=True):
    self.problems=[]
    if not DOMAIN_PATTERN.match(self.domain):
      self.problems.append("Invalid format")
      return
    if PUNYCODE_PATTERN.match(self.domain):
      self.problems.append("Punycode not permitted")
      return
    if not self.extracted:
      self.problems.append("Unparseable domain name")
      return

    if withMalCheck and not self.safebrowsingDate:
      malProblems = malicious_url_check.malCheck(self.extracted,
          writer=DummyFile())
      self.problems.extend(malProblems)
      self.safebrowsingDate = datetime.now()

  def getRegisteredDomain(self):
    return self.extracted.registered_domain


class DummyFile(object):
    def write(self, x): pass

class DomainTester(object):
  def __init__(self):
    self.useGoogle = True
    self.registeredDomains = {}
    self.emailList = {}
    self.invalidList = []
    self.malList = []
    self.shelf = None

  def setUseGoogle(self, value):
    self.useGoogle = value

  def associateDomainWithEmail(self, domainEntry):
    if domainEntry.email not in self.emailList:
      self.emailList[domainEntry.email] = set()

    self.emailList[domainEntry.email].add(domainEntry.domain)

  def loadShelf(self, shelf):
    self.shelf = shelf

    for domain, domainObj in shelf.iteritems():
      assert type(domainObj) is DomainEntry
      self.checkAndTally(domainObj)

  def getOrCreateDomainEntry(self, domain=None, email=None):
    if domain not in self.shelf:
      try:
        extracted = tldextract.extract(domain)
      except ValueError, ve:
        print("Couldn't decode domain {0}: {1}".format(domain, ve))
        raise

      self.shelf[domain] = DomainEntry(extracted, domain=domain, email=email)

    return self.shelf[domain]

  def getDomain(self, domainName):
    obj = self.shelf[domainName]
    assert type(obj) is DomainEntry
    return obj

  def checkAndTally(self, domainEntry):
    domainEntry.check(withMalCheck=self.useGoogle)

    if domainEntry.hasProblems():
      if domainEntry.isInvalid():
        self.invalidList.append(domainEntry)
      else:
        self.malList.append(domainEntry)
    else:
      self.associateDomainWithEmail(domainEntry)
      self.registeredDomains[domainEntry.getRegisteredDomain()] = domainEntry.email

  def getWwwComplementName(self, domainObj):
      assert type(domainObj) is DomainEntry
      if domainObj.extracted.subdomain == "www":
        return domainObj.extracted.registered_domain
      if len(domainObj.extracted.subdomain) < 1:
        return "www.{0}".format(domainObj.extracted.registered_domain)
      return None

  def processEntry(self, domains=[], email=""):
    for domain in re.split('[,; ]', domains):
      domain = domain.lower()
      domain = re.sub("[ ]*(http|https)://", "", domain)
      domain = re.sub("\*\.", "", domain)
      domain = domain.strip()

      if len(domain) < 2:
        continue

      try:
        domainEntry = self.getOrCreateDomainEntry(domain=domain, email=email)
        domainEntry.check()

        if domainEntry.check():
          self.invalidList.append(domainEntry)

        self.checkAndTally(domainEntry)
        if domainEntry.problems:
          continue

        # If it has a WWW, get the not-WWW form. Or add a WWW.
        compDomain = self.getWwwComplementName(domainEntry)
        if compDomain:
          compEntry = self.getOrCreateDomainEntry(domain=compDomain, email=email)
          self.checkAndTally(compEntry)

      except (KeyboardInterrupt, SystemExit):
        raise
      except Exception, err:
        print("Caught exception at {0}: {1}".format(domainEntry, err))

  def listObjectsByDomain(self):<commit_msg>Strip "." and "/" characters from the ends of domains.<commit_after>#!/usr/bin/python
"""
Convert a CSV file into our Beta Whitelist format
"""

try:
  import tldextract
except:
  print("You need to 'pip install -r requirements.txt'")
  sys.exit(1)

import csv, sys, os, re, time, random, shelve
from jinja2 import Template

import malicious_url_check
import argparse
import smtplib

from email.mime.text import MIMEText
from email.MIMEImage import MIMEImage
from email.mime.multipart import MIMEMultipart
from datetime import datetime

# The heading values in the header row for the columns
COL_DOMAIN = "What domain(s) would you like to get a certificate for?"
COL_EMAIL = "What email address can we contact you at when we're ready for you?"

TEMPLATE_DIR = "{0}/templates".format(os.path.dirname(os.path.abspath(__file__)))
DOMAIN_PATTERN = re.compile("^((?!-)[A-Za-z0-9-]{1,63}(?<!-)\.)+[A-Za-z]{2,18}$")
PUNYCODE_PATTERN = re.compile("xn--")

with open(TEMPLATE_DIR+'/btn_donateCC_LG.gif', 'rb') as donateImg:
  DONATE_IMAGE = MIMEImage(donateImg.read())
  DONATE_IMAGE.add_header('Content-ID', '<image1>')

class DomainEntry(object):
  def __init__(self, extracted, domain="", email=""):
    self.extracted = extracted
    self.domain = domain
    self.email = email
    self.problems = []
    self.addedDate = None
    self.safebrowsingDate = None
    self.notificationDate = None

  def __repr__(self):
    msg = "domain={0} email={1}".format(self.domain, self.email)

    if self.problems:
      msg = "{0} problems={1}".format(msg, self.problems)

    if self.safebrowsingDate:
      msg = "{0} S".format(msg)

    if self.addedDate:
      msg = "{0} A".format(msg)

    if self.notificationDate:
      msg = "{0} N".format(msg)

    return msg

  def isInvalid(self):
    if not DOMAIN_PATTERN.match(self.domain):
      return True
    if PUNYCODE_PATTERN.match(self.domain):
      return True
    return False

  def hasProblems(self):
    return len(self.problems) > 0

  def check(self, withMalCheck=True):
    self.problems=[]
    if not DOMAIN_PATTERN.match(self.domain):
      self.problems.append("Invalid format")
      return
    if PUNYCODE_PATTERN.match(self.domain):
      self.problems.append("Punycode not permitted")
      return
    if not self.extracted:
      self.problems.append("Unparseable domain name")
      return

    if withMalCheck and not self.safebrowsingDate:
      malProblems = malicious_url_check.malCheck(self.extracted,
          writer=DummyFile())
      self.problems.extend(malProblems)
      self.safebrowsingDate = datetime.now()

  def getRegisteredDomain(self):
    return self.extracted.registered_domain


class DummyFile(object):
    def write(self, x): pass

class DomainTester(object):
  def __init__(self):
    self.useGoogle = True
    self.registeredDomains = {}
    self.emailList = {}
    self.invalidList = []
    self.malList = []
    self.shelf = None

  def setUseGoogle(self, value):
    self.useGoogle = value

  def associateDomainWithEmail(self, domainEntry):
    if domainEntry.email not in self.emailList:
      self.emailList[domainEntry.email] = set()

    self.emailList[domainEntry.email].add(domainEntry.domain)

  def loadShelf(self, shelf):
    self.shelf = shelf

    for domain, domainObj in shelf.iteritems():
      assert type(domainObj) is DomainEntry
      self.checkAndTally(domainObj)

  def getOrCreateDomainEntry(self, domain=None, email=None):
    if domain not in self.shelf:
      try:
        extracted = tldextract.extract(domain)
      except ValueError, ve:
        print("Couldn't decode domain {0}: {1}".format(domain, ve))
        raise

      self.shelf[domain] = DomainEntry(extracted, domain=domain, email=email)

    return self.shelf[domain]

  def getDomain(self, domainName):
    obj = self.shelf[domainName]
    assert type(obj) is DomainEntry
    return obj

  def checkAndTally(self, domainEntry):
    domainEntry.check(withMalCheck=self.useGoogle)

    if domainEntry.hasProblems():
      if domainEntry.isInvalid():
        self.invalidList.append(domainEntry)
      else:
        self.malList.append(domainEntry)
    else:
      self.associateDomainWithEmail(domainEntry)
      self.registeredDomains[domainEntry.getRegisteredDomain()] = domainEntry.email

  def getWwwComplementName(self, domainObj):
      assert type(domainObj) is DomainEntry
      if domainObj.extracted.subdomain == "www":
        return domainObj.extracted.registered_domain
      if len(domainObj.extracted.subdomain) < 1:
        return "www.{0}".format(domainObj.extracted.registered_domain)
      return None

  def processEntry(self, domains=[], email=""):
    for domain in re.split('[,; ]', domains):
      domain = domain.lower()
      domain = re.sub("[ ]*(http|https)://", "", domain)
      domain = re.sub("\*\.", "", domain)
      domain = re.sub("[\/\.]$", "", domain)
      domain = domain.strip()

      if len(domain) < 2:
        continue

      try:
        domainEntry = self.getOrCreateDomainEntry(domain=domain, email=email)
        domainEntry.check()

        if domainEntry.check():
          self.invalidList.append(domainEntry)

        self.checkAndTally(domainEntry)
        if domainEntry.problems:
          continue

        # If it has a WWW, get the not-WWW form. Or add a WWW.
        compDomain = self.getWwwComplementName(domainEntry)
        if compDomain:
          compEntry = self.getOrCreateDomainEntry(domain=compDomain, email=email)
          self.checkAndTally(compEntry)

      except (KeyboardInterrupt, SystemExit):
        raise
      except Exception, err:
        print("Caught exception at {0}: {1}".format(domainEntry, err))

  def listObjectsByDomain(self):
{'fraction_non_alphanumeric': 0.11761584718783162, 'fraction_numerical': 0.004775380261761585, 'mean_word_length': 3.3446023818670763, 'pattern_counts': {'":': 2, '<': 11, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4183612', 'n_tokens_mistral': 3712, 'n_tokens_neox': 3521, 'n_words': 819}