<commit_before>#!/usr/bin/env python3

# Fetch my fddb food diary

from contextlib import contextmanager
import datetime
import json
import os
import sys
import requests
import re

import dateutil.parser
import lxml.html


def main(argv):
    session = login()
    startdate = dateutil.parser.parse(argv[1])
    lastdate = datetime.datetime.now()
    date = startdate
    days = []
    with login() as session:
        while (date <= lastdate):
            page = page_for_date(date, session)
            day = scrape_day(page, date)
            days.append(day)
            date = next_day(date)


@contextmanager
def login():
    loginpage = 'https://fddb.info/db/i18n/account/?lang=de&action=login'
    user_pass = {}
    with open(os.path.expanduser('~/secret/login-fddb.json')) as f:
        user_pass = json.load(f);
    with requests.Session() as session:
        session.post(loginpage,
                     {
                         'loginemailorusername': user_pass['user'],
                         'loginpassword': <PASSWORD>['password']
                     });
        yield session


def page_for_date(date, session):
    url = url_for_day(date)
    content = session.get(url).content
    page = lxml.html.document_fromstring(content)
    return page


def url_for_day(date):
    fddb_date_page_format = 'http://fddb.info/db/i18n/myday20/?lang=de&q={end}&p={start}'
    nextdate = next_day(date)
    start = int(date.timestamp())
    end = int(nextdate.timestamp())
    url = fddb_date_page_format.format(start=start, end=end)
    return url


def scrape_day(page, date):
    foods = scrape_foods(page, date)
    day = dict(
        date=date,
        total=scrape_total(page),
        foods=foods)
    return day


def scrape_total(page):
    table = page.xpath('//p[@id="fddb-myday-printinfo"]/following-sibling::table[1]')[0]
    return {'kcal':
            parse_kcal(table.xpath('.//tr[1]/td[2]/text()')[0]),
            # every following xpath is identical after tr[n], but kcal
            # must really be differen or it won't match (kJ would match, though)
            'fat': parse_g(table.xpath('.//tr[2]/td[2]//text()')[0]),
            'carbs': parse_g(table.xpath('.//tr[3]/td[2]//text()')[0]),
            'sugar': parse_g(table.xpath('.//tr[4]/td[2]//text()')[0]),
            'protein': parse_g(table.xpath('.//tr[5]/td[2]//text()')[0]),
            'alcohol': parse_g(table.xpath('.//tr[6]/td[2]//text()')[0]),
            'water': parse_liters(table.xpath('.//tr[7]/td[2]//text()')[0]),
            'fibre': parse_g(table.xpath('.//tr[8]/td[2]//text()')[0]),
            'cholesterol': parse_mg(table.xpath('.//tr[9]/td[2]//text()')[0]),
            'BE': parse_german_float(table.xpath('.//tr[10]/td[2]//text()')[0])}


def next_day(date):
    return date + datetime.timedelta(days=1)

def scrape_foods(page, date):
    food_rows = page.xpath('//table[@class="myday-table-std"]//tr')
    foods = []
    for row in food_rows:
        time_el = row.xpath('td[1]/span[@class="mydayshowtime"]/text()')
        if not time_el:
            continue
        time = dateutil.parser.parse(time_el[0], default=date)
        food = row.xpath('td[1]/a')[0]
        food_link = food.attrib['href']
        food_txt = food.text
        kcal = parse_kcal(row.xpath('td[3]//text()')[0])
        fat = parse_g(row.xpath('td[4]//text()')[0])<commit_msg>make xpath expressions shorter (and identical) by operating on trs list<commit_after>#!/usr/bin/env python3

# Fetch my fddb food diary

from contextlib import contextmanager
import datetime
import json
import os
import sys
import requests
import re

import dateutil.parser
import lxml.html


def main(argv):
    session = login()
    startdate = dateutil.parser.parse(argv[1])
    lastdate = datetime.datetime.now()
    date = startdate
    days = []
    with login() as session:
        while (date <= lastdate):
            page = page_for_date(date, session)
            day = scrape_day(page, date)
            days.append(day)
            date = next_day(date)


@contextmanager
def login():
    loginpage = 'https://fddb.info/db/i18n/account/?lang=de&action=login'
    user_pass = {}
    with open(os.path.expanduser('~/secret/login-fddb.json')) as f:
        user_pass = json.load(f);
    with requests.Session() as session:
        session.post(loginpage,
                     {
                         'loginemailorusername': user_pass['user'],
                         'loginpassword': user_pass['password']
                     });
        yield session


def page_for_date(date, session):
    url = url_for_day(date)
    content = session.get(url).content
    page = lxml.html.document_fromstring(content)
    return page


def url_for_day(date):
    fddb_date_page_format = 'http://fddb.info/db/i18n/myday20/?lang=de&q={end}&p={start}'
    nextdate = next_day(date)
    start = int(date.timestamp())
    end = int(nextdate.timestamp())
    url = fddb_date_page_format.format(start=start, end=end)
    return url


def scrape_day(page, date):
    foods = scrape_foods(page, date)
    day = dict(
        date=date,
        total=scrape_total(page),
        foods=foods)
    return day


def scrape_total(page):
    trs = page.xpath('//p[@id="fddb-myday-printinfo"]/following-sibling::table[1]//tr')
    return {'kcal':
            parse_kcal(trs[0].xpath('.//td[2]/text()')[0]),
            # every following xpath is identical after tr[n], but kcal
            # must really be different or it won't match (kJ would match, though)
            'fat': parse_g(trs[1].xpath('./td[2]//text()')[0]),
            'carbs': parse_g(trs[2].xpath('.//td[2]//text()')[0]),
            'sugar': parse_g(trs[3].xpath('.//td[2]//text()')[0]),
            'protein': parse_g(trs[4].xpath('.//td[2]//text()')[0]),
            'alcohol': parse_g(trs[5].xpath('.//td[2]//text()')[0]),
            'water': parse_liters(trs[6].xpath('.//td[2]//text()')[0]),
            'fibre': parse_g(trs[7].xpath('.//td[2]//text()')[0]),
            'cholesterol': parse_mg(trs[8].xpath('.//td[2]//text()')[0]),
            'BE': parse_german_float(trs[9].xpath('.//td[2]//text()')[0])}


def next_day(date):
    return date + datetime.timedelta(days=1)

def scrape_foods(page, date):
    food_rows = page.xpath('//table[@class="myday-table-std"]//tr')
    foods = []
    for row in food_rows:
        time_el = row.xpath('td[1]/span[@class="mydayshowtime"]/text()')
        if not time_el:
            continue
        time = dateutil.parser.parse(time_el[0], default=date)
        food = row.xpath('td[1]/a')[0]
        food_link = food.attrib['href']
        food_txt = food.text
        kcal = parse_kcal(row.xpath('td[3]//text()')[0])
        fat = parse_g(row.xpath('td[4]//text()')[0])
{'fraction_non_alphanumeric': 0.17746478873239438, 'fraction_numerical': 0.014529280948851002, 'mean_word_length': 2.642548596112311, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 4, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '870090', 'n_tokens_mistral': 2409, 'n_tokens_neox': 2339, 'n_words': 401}