<commit_before>"""
SiteAlert, what are you waiting for?

Copyright (c) 2015, <NAME> <---> ilteoood
All rights reserved.

Redistribution and use in source and binary forms, with or without modification, are permitted provided
that the following conditions are met:

* Redistributions of source code must retain the above copyright notice, this list of conditions and the
  following disclaimer.

* Redistributions in binary form must reproduce the above copyright notice, this list of conditions and
  the following disclaimer in the documentation and/or other materials provided with the distribution.

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
========================================================================================================================
"""
__author__ = 'iLTeoooD'
import urllib.request
import hashlib
import sqlite3
import os
import smtplib
import time
import sys
import platform
from os.path import expanduser

import telebot
from bs4 import BeautifulSoup

db = expanduser("~") + os.sep + "SiteAlert.db"
header = [('User-Agent',
           'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11'),
          ('Accept', 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8'),
          ('Accept-Charset', 'ISO-8859-1,utf-8;q=0.7,*;q=0.3'),
          ('Accept-Encoding', 'none'),
          ('Accept-Language', 'en-US,en;q=0.8'),
          ('Connection', 'keep-alive')]
TOKEN = 'YOUR TOKEN HERE'
MAIL = 'YOUR MAIL HERE'
PSW = 'YOUR <PASSWORD>'
tb = telebot.TeleBot(TOKEN)


def clearScreen():
    if platform.system() == "Windows":
        os.system("cls")
    else:
        os.system("clear")


def visualizeMenu():
    clearScreen()
    print(
        "What do you want to do?\n1) Display sites\n2) Add new site to check\n3) Fetch site\n4) Check sites\n5) Add e-mail to notification\n6) Remove e-mail from notification\n7) Delete a site\n8) Clean database\n9) Exit")


def choice():
    clearScreen()
    try:
        x = -1
        while not 1 <= x <= 9:
            if x != 9:
                visualizeMenu()
            x = int(input())
        return x
    except ValueError:
        return 9


def displaySites(dirs, leng):
    if leng != 0:
        i = 1
        for dir in dirs:
            print(str(i) + ") " + dir[0])
            i += 1
    else:
        print("You haven't checked any site!")


def cleanDB(f):
    nameSite = f.execute("SELECT name FROM SiteAlert EXCEPT SELECT name FROM Registered GROUP BY name").fetchall()
    for name in nameSite:
        print("Removing %s..." % (name))
        f.execute("DELETE FROM SiteAlert WHERE name = \"%s\"" % (name))
    f.commit()


def stdURL(site):
    if not site.startswith("http://") and not site.startswith("https://"):
        site = "http://" + site
    return site


def URLEncode(read):
    read = BeautifulSoup(read, "html.parser")
    return hashlib.md5(bytes(read.get_text(), 'utf-8')).hexdigest()


def saveFile(f, nameSite, link, mail, telegram, hash):
    try:
        f.execute("INSERT INTO SiteAlert (name,link,hash) VALUES (\"%s\",\"%s\",\"%s\")" % (
            nameSite, link, hash))
        mail = mail.split(";");
        for m in mail:
            f.execute("INSERT INTO Registered (name, mail) VALUES (\"%s\",\"%s\")" % (
                nameSite, m))
    except sqlite3.IntegrityError:
        f.execute("UPDATE SiteAlert SET hash=\"%s\" WHERE name=\"%s\"" % (hash, nameSite))
    f.commit()
    print("Site saved correctly!")


def addSite(f, nameSite, link, mail='', telegram=''):
    if link == "" or nameSite == "":
        link = input("Insert the link for the site: ")
        nameSite = input("Insert a name for the site: ")
        mail = input(
            "Insert the email where you want to be informed (if you want to add other mail, separate them with \";\"): ")
    try:
        link = stdURL(link)
        urli = urllib.request.build_opener()
        urli.addheaders = header
        urli = urli.open(link)
        responseCode = urli.getcode()
        if responseCode == 200:
            saveFile(f, nameSite, link, mail, telegram, URLEncode(urli.read()))
        elif responseCode == 404:
            print("This page doesn't exist!")
        else:
            print("Generic error.")
    except urllib.request.URLError:
        print("There is an error with the link.")


def sendMail(f, nameSite, link):
    try:
        global MAIL, PSW
        server = smtplib.SMTP("smtp.gmail.com:587")
        server.starttls()
        server.login(MAIL, PSW)
        subj = "The site \"" + nameSite + "\" has been changed!"
        msg = "Subject: " + subj + "\n" + subj + "\nLink: " + link
        mail = f.execute("SELECT mail FROM Registered WHERE name=\"%s\"" % (nameSite)).fetchall()
        for address in mail:
            server.sendmail(MAIL, address, msg)
            t = f.execute("SELECT telegram FROM Users, Registered WHERE name=\"%s\" AND Users.mail = Registered.mail" % (nameSite)).fetchone()
            tb.send_message(t[0], subj + "\nLink: " + link)
        server.close()
    except smtplib.SMTPRecipientsRefused:
        print("Error with the e-mail destination address.")
    except smtplib.SMTPAuthenticationError:
        print("Error in the login process")


def checkSite(f, dirs):
    if len(dirs) > 0:
        for dir in dirs:
            dir = dir[0]
            query = f.execute("SELECT hash,link FROM SiteAlert WHERE name=\"" + dir + "\"").fetchone()
            hash = query[0]<commit_msg>Better change recognition<commit_after>"""
SiteAlert, what are you waiting for?

Copyright (c) 2015, <NAME> <---> ilteoood
All rights reserved.

Redistribution and use in source and binary forms, with or without modification, are permitted provided
that the following conditions are met:

* Redistributions of source code must retain the above copyright notice, this list of conditions and the
  following disclaimer.

* Redistributions in binary form must reproduce the above copyright notice, this list of conditions and
  the following disclaimer in the documentation and/or other materials provided with the distribution.

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
========================================================================================================================
"""
__author__ = 'iLTeoooD'
import urllib.request
import hashlib
import sqlite3
import os
import smtplib
import time
import sys
import platform
import re
from os.path import expanduser

import telebot
from bs4 import BeautifulSoup

db = expanduser("~") + os.sep + "SiteAlert.db"
header = [('User-Agent',
           'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11'),
          ('Accept', 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8'),
          ('Accept-Charset', 'ISO-8859-1,utf-8;q=0.7,*;q=0.3'),
          ('Accept-Encoding', 'none'),
          ('Accept-Language', 'en-US,en;q=0.8'),
          ('Connection', 'keep-alive')]
TOKEN = 'YOUR TOKEN HERE'
MAIL = 'YOUR MAIL HERE'
PSW = 'YOUR PASSWORD HERE'
tb = telebot.TeleBot(TOKEN)


def clearScreen():
    if platform.system() == "Windows":
        os.system("cls")
    else:
        os.system("clear")


def visualizeMenu():
    clearScreen()
    print(
        "What do you want to do?\n1) Display sites\n2) Add new site to check\n3) Fetch site\n4) Check sites\n5) Add e-mail to notification\n6) Remove e-mail from notification\n7) Delete a site\n8) Clean database\n9) Exit")


def choice():
    clearScreen()
    try:
        x = -1
        while not 1 <= x <= 9:
            if x != 9:
                visualizeMenu()
            x = int(input())
        return x
    except ValueError:
        return 9


def displaySites(dirs, leng):
    if leng != 0:
        i = 1
        for dir in dirs:
            print(str(i) + ") " + dir[0])
            i += 1
    else:
        print("You haven't checked any site!")


def cleanDB(f):
    nameSite = f.execute("SELECT name FROM SiteAlert EXCEPT SELECT name FROM Registered GROUP BY name").fetchall()
    for name in nameSite:
        print("Removing %s..." % (name))
        f.execute("DELETE FROM SiteAlert WHERE name = \"%s\"" % (name))
    f.commit()


def stdURL(site):
    if not site.startswith("http://") and not site.startswith("https://"):
        site = "http://" + site
    return site


def URLEncode(read):
    read = BeautifulSoup(read, "html.parser")
    read = re.sub("<!--[\s\S]*?-->", "", read.get_text())
    read = re.sub("(?s)/\\*.*?\\*/", "", read)
    return hashlib.md5(bytes(read, 'utf-8')).hexdigest()


def saveFile(f, nameSite, link, mail, telegram, hash):
    try:
        f.execute("INSERT INTO SiteAlert (name,link,hash) VALUES (\"%s\",\"%s\",\"%s\")" % (
            nameSite, link, hash))
        mail = mail.split(";");
        for m in mail:
            f.execute("INSERT INTO Registered (name, mail) VALUES (\"%s\",\"%s\")" % (
                nameSite, m))
    except sqlite3.IntegrityError:
        f.execute("UPDATE SiteAlert SET hash=\"%s\" WHERE name=\"%s\"" % (hash, nameSite))
    f.commit()
    print("Site saved correctly!")


def addSite(f, nameSite, link, mail='', telegram=''):
    if link == "" or nameSite == "":
        link = input("Insert the link for the site: ")
        nameSite = input("Insert a name for the site: ")
        mail = input(
            "Insert the email where you want to be informed (if you want to add other mail, separate them with \";\"): ")
    try:
        link = stdURL(link)
        urli = urllib.request.build_opener()
        urli.addheaders = header
        urli = urli.open(link)
        responseCode = urli.getcode()
        if responseCode == 200:
            saveFile(f, nameSite, link, mail, telegram, URLEncode(urli.read()))
        elif responseCode == 404:
            print("This page doesn't exist!")
        else:
            print("Generic error.")
    except urllib.request.URLError:
        print("There is an error with the link.")


def sendMail(f, nameSite, link):
    try:
        global MAIL, PSW
        server = smtplib.SMTP("smtp.gmail.com:587")
        server.starttls()
        server.login(MAIL, PSW)
        subj = "The site \"" + nameSite + "\" has been changed!"
        msg = "Subject: " + subj + "\n" + subj + "\nLink: " + link
        mail = f.execute("SELECT mail FROM Registered WHERE name=\"%s\"" % (nameSite)).fetchall()
        for address in mail:
            server.sendmail(MAIL, address, msg)
            t = f.execute(
                "SELECT telegram FROM Users, Registered WHERE name=\"%s\" AND Users.mail = Registered.mail" % (
                nameSite)).fetchone()
            tb.send_message(t[0], subj + "\nLink: " + link)
        server.close()
    except smtplib.SMTPRecipientsRefused:
        print("Error with the e-mail destination address.")
    except smtplib.SMTPAuthenticationError:
        print("Error in the login process")


def checkSite(f, dirs):
    if len(dirs) > 0:
        for dir in dirs:
            dir = dir[0]
            query = f.execute("SELECT hash,link FROM SiteAlert WHERE name=\"" + dir + "\"").fetchone()
            hash = query[0]
{'fraction_non_alphanumeric': 0.14392315630705724, 'fraction_numerical': 0.013177740731920299, 'mean_word_length': 2.8210494388838336, 'pattern_counts': {'":': 4, '<': 13, '<?xml version=': 0, '>': 11, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1913157', 'n_tokens_mistral': 3951, 'n_tokens_neox': 3716, 'n_words': 1273}