<commit_before>import bs4
import requests
import subprocess
import tempfile

from canteens.canteen import Canteen, VEGGIE, MEAT


def __parse_menu(url):

    def get_dropbox_link():
        request = requests.get('http://cafenero.net/speisen.html')
        if request.status_code == requests.codes.ok:
            soup = bs4.BeautifulSoup(request.text, 'html.parser')
            iframe = soup.find('iframe')
            return iframe.attrs['src']
        else:
            return False

    def get_pdf():
        dropbox_link = get_dropbox_link()
        if dropbox_link:
            request = requests.get(dropbox_link)
            if request.status_code == requests.codes.ok:
                tmpdir = tempfile.mkdtemp()
                pdfpath = '%s/cafenero.pdf' % tmpdir
                with open(pdfpath, 'wb') as f:
                    f.write(request.content)
                return tmpdir
            else:
                return False
        else:
            return False

    def pdf_to_text(tmpdir):
        pdfpath = '%s/cafenero.pdf' % tmpdir
        txtpath = '%s/cafenero.txt' % tmpdir
        popen = subprocess.Popen(('pdftotext', '-layout', pdfpath), stdout=subprocess.PIPE)
        popen.wait()
        with open(txtpath, 'r') as f:
            menu = f.read()
        return menu

    return pdf_to_text(get_pdf())

cafenero = Canteen(
    id_='tu_cafenero',
    name='<NAME>',
    url='',
    update=__parse_menu,
    website='http://cafenero.net'
)

CANTEENS = [cafenero]

if __name__ == '__main__':
    print(__parse_menu())
<commit_msg>[cafenero] parse cafenero pdf (fix #22)<commit_after>import bs4
import requests
import subprocess
import tempfile

from canteens.canteen import Canteen, VEGGIE, MEAT


def __parse_menu(url):

    def get_dropbox_link():
        request = requests.get('http://cafenero.net/speisen.html')
        if request.status_code == requests.codes.ok:
            soup = bs4.BeautifulSoup(request.text, 'html.parser')
            iframe = soup.find('iframe')
            return iframe.attrs['src']
        else:
            return False

    def get_pdf():
        dropbox_link = get_dropbox_link()
        if dropbox_link:
            request = requests.get(dropbox_link)
            if request.status_code == requests.codes.ok:
                tmpdir = tempfile.mkdtemp()
                pdfpath = '%s/cafenero.pdf' % tmpdir
                with open(pdfpath, 'wb') as f:
                    f.write(request.content)
                return tmpdir
            else:
                return False
        else:
            return False

    def pdf_to_text(tmpdir):
        pdfpath = '%s/cafenero.pdf' % tmpdir
        txtpath = '%s/cafenero.txt' % tmpdir
        popen = subprocess.Popen(('pdftotext', '-layout', pdfpath), stdout=subprocess.PIPE)
        popen.wait()
        with open(txtpath, 'r') as f:
            menu = f.read()
        return menu

    def get_menu():
        def clean_newlines(text):
            cleaned_result = ''
            for line in text.splitlines():
                if not line == '' and 'cafeneroinder' not in line:
                    if 'mittagstisch' in line or '---' in line or 'schonkost' in line:
                        line = '%s\n' % line
                    else:
                        if line.endswith(('€', 'vegetarisch', 'schweinefleisch')):
                            line = '%s\n' % line
                        else:
                            line = '%s ' % line
                    # print(line)
                    cleaned_result += line
                # print('=' * 30)
            return cleaned_result

        result = ''
        for entry in clean_newlines(pdf_to_text(get_pdf())).splitlines():
            if 'mittagstisch' in entry or '---' in entry or 'schonkost' in entry:
                annotation = ''
            elif 'vegetarisch' in entry:
                annotation = '%s ' % VEGGIE
            else:
                annotation = '%s ' % MEAT
            entry = entry.replace('vegetarisch', '')
            entry = entry.replace('schweinefleisch', '')
            entry = ' '.join(entry.split())
            result += '%s%s\n' % (annotation, entry)
        return result

    return get_menu()

cafenero = Canteen(
    id_='tu_cafenero',
    name='<NAME>',
    url='',
    update=__parse_menu,
    website='http://cafenero.net'
)

CANTEENS = [cafenero]

if __name__ == '__main__':
    print(__parse_menu(''))

{'fraction_non_alphanumeric': 0.11652017128690556, 'fraction_numerical': 0.001803020058598152, 'mean_word_length': 1.7479876160990713, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6137441', 'n_tokens_mistral': 1406, 'n_tokens_neox': 1329, 'n_words': 297}