<commit_before>"""Trenitalia _result parser"""

from bs4 import BeautifulSoup
import re

COMFORTS = ["STANDARD", "PREMIUM", "BUISNESS SALOTTINO",
            "BUISNESS AREA SILENZIO", "BUISNESS", "EXECUTIVE"]

class TrenitaliaParser(object):
    """Trenitalia result parser Class"""
    def __init__(self, html):
        self._result = list()
        self._soup = BeautifulSoup(html)

    def get_results(self):
        """parse trenitalia html result"""

        table = self._soup.find("table", class_="searchResult")
        for res_tr in table.find_all("tr"):
            if "id" in res_tr.attrs and res_tr.attrs["id"].startswith("trow"):
                tds = res_tr.find_all("td")
                dep_td = tds[0]
                arr_td = tds[1]
                dur_td = tds[2]
                train_td = tds[3]
                price_td = tds[5]
                opt = dict()

                tmp = re.sub(r'[\t\n]*', '',
                             train_td.div.span.getText()).split()

                # TODO: different train types have a different structure
                # at the moment parse only FRECCIAROSSA trains
                if tmp[0] != "FRECCIAROSSA":
                    continue
  
                opt["trainType"] = tmp[0]

                opt["depTime"] = dep_td.span.string
                opt["arrTime"] = arr_td.span.string
                opt["duration"] = dur_td.span.string
                tmp = re.sub(r'[\t\n]*', '',
                             train_td.div.span.getText()).split()
                opt["trainType"] = tmp[0]
                opt["trainCode"] = tmp[1]
                opt["minPrice"] = re.sub(r'[\s]', '',
                                         price_td.span.get_text())[:-1]
                opt["prices"] = list()
                for sib in res_tr.find_next_siblings("tr"):
                    if sib.has_attr("id") and sib["id"].startswith("sd_"):
                        pr_tab = sib.table
                        for idx, price_tr in enumerate(pr_tab.find_all("tr")):
                            # skip first two rows
                            if idx <= 1:
                                continue

                            pr_res = dict()
                            tds = price_tr.find_all("td")

                            if not (tds[0].span and tds[0].span.string):
                                continue

                            # BASE, ECONOMY ...
                            pr_res["fareType"] = tds[0].span.string.strip()
                            pr_res["farePrices"] = list()
                            for i in range(1, 7):
                                if not tds[i].span:
                                    tmp = "none"
                                else:
                                    tmp = re.sub(r'[\s]', '',
                                                 tds[i].span.get_text())
                                    if tmp == "SoldOut":
                                        tmp = "none"
                                    else:
                                        tmp = tmp[:-1]
                                pr_res["farePrices"].append({
                                                    "comfort": COMFORTS[i-1],
                                                    "price": tmp})

                            opt["prices"].append(pr_res)
                        break
                self._result.append(opt)<commit_msg>stylistic changes<commit_after>"""Trenitalia _result parser"""

from bs4 import BeautifulSoup
import re

COMFORTS = ["STANDARD", "PREMIUM", "BUISNESS SALOTTINO",
            "BUISNESS AREA SILENZIO", "BUISNESS", "EXECUTIVE"]

class TrenitaliaParser(object):
    """Trenitalia result parser Class"""
    def __init__(self, html):
        self._result = list()
        self._soup = BeautifulSoup(html)

    def get_results(self):
        """parse trenitalia html result"""

        # result table
        table = self._soup.find("table", class_="searchResult")
        for res_tr in table.find_all("tr"):
            if not ("id" in res_tr.attrs and res_tr["id"].startswith("trow")):
                continue
            tds = res_tr.find_all("td")
            dep_td = tds[0]
            arr_td = tds[1]
            dur_td = tds[2]
            train_td = tds[3]
            price_td = tds[5]
            opt = dict()

            tmp = re.sub(r'[\t\n]*', '',
                         train_td.div.span.getText()).split()

            # TODO: different train types have a different structure
            # at the moment parse only FRECCIAROSSA trains
            if tmp[0] != "FRECCIAROSSA":
                continue

            opt["trainType"] = tmp[0]

            opt["depTime"] = dep_td.span.string
            opt["arrTime"] = arr_td.span.string
            opt["duration"] = dur_td.span.string
            tmp = re.sub(r'[\t\n]*', '',
                         train_td.div.span.getText()).split()
            opt["trainType"] = tmp[0]
            opt["trainCode"] = tmp[1]
            opt["minPrice"] = re.sub(r'[\s]', '',
                                     price_td.span.get_text())[:-1]
            opt["prices"] = list()
            for sib in res_tr.find_next_siblings("tr"):
                if not ("id" in sib.attrs and sib["id"].startswith("sd_")):
                    continue
                pr_tab = sib.table
                for idx, price_tr in enumerate(pr_tab.find_all("tr")):
                    # skip first two rows
                    if idx <= 1:
                        continue

                    pr_res = dict()
                    tds = price_tr.find_all("td")

                    if not (tds[0].span and tds[0].span.string):
                        continue

                    # BASE, ECONOMY ...
                    pr_res["fareType"] = tds[0].span.string.strip()
                    pr_res["farePrices"] = list()
                    for i in range(1, 7):
                        if not tds[i].span:
                            tmp = "none"
                        else:
                            tmp = re.sub(r'[\s]', '',
                                         tds[i].span.get_text())
                            if tmp == "SoldOut":
                                tmp = "none"
                            else:
                                tmp = tmp[:-1]
                        pr_res["farePrices"].append({
                                            "comfort": COMFORTS[i-1],
                                            "price": tmp})

                    opt["prices"].append(pr_res)
                break
            self._result.append(opt)
{'fraction_non_alphanumeric': 0.1203983101991551, 'fraction_numerical': 0.005733252866626433, 'mean_word_length': 1.0561414392059554, 'pattern_counts': {'":': 8, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 29, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7014832', 'n_tokens_mistral': 1847, 'n_tokens_neox': 1746, 'n_words': 360}