<commit_before>import requests
# from collections import namedtuple
from time import sleep
import xml.etree.ElementTree as ET
from .models import GoodReadsData
from dexer.settings import GOODREADS_KEYS
import random

# GoodReadsBook = namedtuple("GoodReadsBook", [
#     "gr_id", "nr_reviews", "nr_text_reviews", "pub_year", "pub_month", "pub_day",
#     "average_rating", "title", "authors", "author_id", "img", "small_img"
# ])


SEARCH_URL = "https://www.goodreads.com/search/index.xml"


def search(title):
    sleep(1) # api requires us to be nice
    response = requests.get(SEARCH_URL, data={"key": _get_key(), "q": title})
    if response.status_code != 200:
        print(response.status_code, response.content)
        return []

    content = ET.fromstring(response.content)

    search_tag = _get_child(content, "search")
    results_tag = _get_child(search_tag, "results")

    books = []
    for result in results_tag.getchildren():
        # print "\n\n\n\nResult", index
        # _debug_print(result)
        books.append(make_book(result))

    return books


def _get_key():
    return random.choice(GOODREADS_KEYS)

def _debug_print(node, padding=""):
    print(padding, node.tag, node.text, node.attrib)
    padding += "    "
    for child in node.getchildren():
        _debug_print(child, padding)


def make_book(node):
    gr_id = _get_child_val(node, "id")
    nr_reviews = _get_child_val(node, "ratings_count")
    nr_text_reviews = _get_child_val(node, "text_ratings_count")
    pub_year = _get_child_val(node, "original_publication_year")
    pub_month = _get_child_val(node, "original_publication_month")
    pub_day = _get_child_val(node, "original_publication_day")
    average_rating = _get_child_val(node, "average_rating")
    book_info = _get_child(node, "best_book")
    title = _get_child_val(book_info, "title")
    author_data = _get_child(book_info, "author")
    authors = _get_child_val(author_data, "name")
    author_id = _get_child_val(author_data, "id")
    img = _get_child_val(book_info, "image_url")
    small_img = _get_child_val(book_info, "small_image_url")

    book = GoodReadsData(
        good_reads_id=gr_id,
        nr_reviews=nr_reviews,
        nr_text_reviews=nr_text_reviews,
        pub_year=pub_year,
        pub_month=pub_month,
        pub_day=pub_day,
        average_rating=average_rating,
        title=title,
        authors=authors,
        author_id=author_id,
        img=img,
        small_img=small_img
    )
    return book


def _get_child(parent, tag_name):
    for child in parent.getchildren():
        if child.tag == tag_name:
            return child
    return None


def _get_child_val(parent, tag_name, default=None):
    for child in parent.getchildren():
        if child.tag == tag_name:
            return child.text
    return default
<commit_msg>Must use only public key.<commit_after>import requests
# from collections import namedtuple
from time import sleep
import xml.etree.ElementTree as ET
from .models import GoodReadsData
from dexer.settings import GOODREADS_KEYS
import random

# GoodReadsBook = namedtuple("GoodReadsBook", [
#     "gr_id", "nr_reviews", "nr_text_reviews", "pub_year", "pub_month", "pub_day",
#     "average_rating", "title", "authors", "author_id", "img", "small_img"
# ])


SEARCH_URL = "https://www.goodreads.com/search/index.xml"


def search(title):
    sleep(1) # api requires us to be nice
    response = requests.get(SEARCH_URL, data={"key": _get_key(), "q": title})
    if response.status_code != 200:
        print(response.status_code, response.content)
        return []

    content = ET.fromstring(response.content)

    search_tag = _get_child(content, "search")
    results_tag = _get_child(search_tag, "results")

    books = []
    for result in results_tag.getchildren():
        # print "\n\n\n\nResult", index
        # _debug_print(result)
        books.append(make_book(result))

    return books


def _get_key():
    return random.choice(GOODREADS_KEYS)[0]

def _debug_print(node, padding=""):
    print(padding, node.tag, node.text, node.attrib)
    padding += "    "
    for child in node.getchildren():
        _debug_print(child, padding)


def make_book(node):
    gr_id = _get_child_val(node, "id")
    nr_reviews = _get_child_val(node, "ratings_count")
    nr_text_reviews = _get_child_val(node, "text_ratings_count")
    pub_year = _get_child_val(node, "original_publication_year")
    pub_month = _get_child_val(node, "original_publication_month")
    pub_day = _get_child_val(node, "original_publication_day")
    average_rating = _get_child_val(node, "average_rating")
    book_info = _get_child(node, "best_book")
    title = _get_child_val(book_info, "title")
    author_data = _get_child(book_info, "author")
    authors = _get_child_val(author_data, "name")
    author_id = _get_child_val(author_data, "id")
    img = _get_child_val(book_info, "image_url")
    small_img = _get_child_val(book_info, "small_image_url")

    book = GoodReadsData(
        good_reads_id=gr_id,
        nr_reviews=nr_reviews,
        nr_text_reviews=nr_text_reviews,
        pub_year=pub_year,
        pub_month=pub_month,
        pub_day=pub_day,
        average_rating=average_rating,
        title=title,
        authors=authors,
        author_id=author_id,
        img=img,
        small_img=small_img
    )
    return book


def _get_child(parent, tag_name):
    for child in parent.getchildren():
        if child.tag == tag_name:
            return child
    return None


def _get_child_val(parent, tag_name, default=None):
    for child in parent.getchildren():
        if child.tag == tag_name:
            return child.text
    return default

{'fraction_non_alphanumeric': 0.1101352538204813, 'fraction_numerical': 0.0015808888108203056, 'mean_word_length': 3.629268292682927, 'pattern_counts': {'":': 4, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6857985', 'n_tokens_mistral': 1971, 'n_tokens_neox': 1924, 'n_words': 380}