<commit_before>import re

from bs4 import BeautifulSoup
from nltk.corpus import stopwords
import requests
from sqlalchemy.orm import sessionmaker


from models import create_db_tables, db_connect, WebsitesContent
from google import search

MAX_RESULTS_LIMIT = 50


def get_queries():
    """Returns a list of queries we want to index data.
    Note: for now it is a hardcoded list but it will be replaced with data fetched from Twitter.
    """
    return ['cars']


def index_query_data(query, db_session):
    """Fetch google pages for a query and index their content. """
    words = []
    for url in search(query, stop=MAX_RESULTS_LIMIT):
        try:
            words = extract_words_from_url(url)
            db_session.add(WebsitesContent(link=url, words=words))
            db_session.commit()
        except:
            print url

    return words


def extract_words_from_url(url):
    """Use BeautifulSoup to extract visible text from a web page. """
    def visible(element):
        """Method used to filter visible text elements. """
        if element.parent.name in ['style', 'script', '[document]', 'head', 'link']:
            return False
        elif re.match('<!--.*-->', element.encode('utf-8')):
            return False

        value = element.encode('utf-8')
        return not value.isspace()

    html = requests.get(url, verify=False).content
    soup = BeautifulSoup(html, 'html.parser')
    texts = soup.findAll(text=True)

    visible_texts = [text for text in texts if visible(text)]

    words = []
    stop_words = stopwords.words('english')
    for text in visible_texts:
        for word in re.findall(r"[\w']+", text):
            if not word.isdigit() and word.lower() not in stop_words:
                words.append(word)


    return words


def main():
    """Based on a list of queries:
     - get the first 50 results from google
     - scrap their text
     - store text in the db
     - get demographics for page
     - store demographics
    """
    engine = db_connect()
    create_db_tables(engine)
    Session = sessionmaker(bind=engine)
    session = Session()

    quereies = get_queries()

    for query in quereies:
        index_query_data(query, session)

    session.close()


if __name__ == '__main__':
    main()
<commit_msg>All words should be in lower case.<commit_after>import re

from bs4 import BeautifulSoup
from nltk.corpus import stopwords
import requests
from sqlalchemy.orm import sessionmaker


from models import create_db_tables, db_connect, WebsitesContent
from google import search

MAX_RESULTS_LIMIT = 50


def get_queries():
    """Returns a list of queries we want to index data.
    Note: for now it is a hardcoded list but it will be replaced with data fetched from Twitter.
    """
    return ['cars']


def index_query_data(query, db_session):
    """Fetch google pages for a query and index their content. """
    words = []
    for url in search(query, stop=MAX_RESULTS_LIMIT):
        try:
            words = extract_words_from_url(url)
            db_session.add(WebsitesContent(link=url, words=words))
            db_session.commit()
        except:
            print url

    return words


def extract_words_from_url(url):
    """Use BeautifulSoup to extract visible text from a web page. """
    def visible(element):
        """Method used to filter visible text elements. """
        if element.parent.name in ['style', 'script', '[document]', 'head', 'link']:
            return False
        elif re.match('<!--.*-->', element.encode('utf-8')):
            return False

        value = element.encode('utf-8')
        return not value.isspace()

    html = requests.get(url, verify=False).content
    soup = BeautifulSoup(html, 'html.parser')
    texts = soup.findAll(text=True)

    visible_texts = [text for text in texts if visible(text)]

    words = []
    stop_words = stopwords.words('english')
    for text in visible_texts:
        for word in re.findall(r"[\w']+", text):
            word = word.lower()
            if not word.isdigit() and word not in stop_words:
                words.append(word)


    return words


def main():
    """Based on a list of queries:
     - get the first 50 results from google
     - scrap their text
     - store text in the db
     - get demographics for page
     - store demographics
    """
    engine = db_connect()
    create_db_tables(engine)
    Session = sessionmaker(bind=engine)
    session = Session()

    quereies = get_queries()

    for query in quereies:
        index_query_data(query, session)

    session.close()


if __name__ == '__main__':
    main()

{'fraction_non_alphanumeric': 0.10088763801688677, 'fraction_numerical': 0.0030309590820523924, 'mean_word_length': 2.675417661097852, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3093122', 'n_tokens_mistral': 1378, 'n_tokens_neox': 1323, 'n_words': 446}