<commit_before>defmodule Data do

  @max_respone_rows 300

  def main(args) do
    args |> parse_args |> process
  end

  def process([]) do
    IO.puts "No arguments given"
  end

  def process([type: "html", year: year, round: round]) do
    data_filename = "dd-f1-#{year}-#{round}.json"
    File.write("dd-f1-#{year}-#{round}.html", EEx.eval_file("laps.html.eex", [year: year, round: round, data_src: data_filename]))
    #File.write("dd-f1-qualy-#{year}-#{round}.html", EEx.eval_file("qualy.html.eex", [year: year, round: round, data_src: data_filename]))
  end


  def process([type: "all", year: year, round: round]) do
    #get_tire_data(year, round)

    # get tire data:
    tire_data = with_cached_file "dd-#{year}-#{round}-tires.json", fn() -> get_tire_data(year, round) end

    # get race results
    results = with_cached_file "dd-#{year}-#{round}-results.json", fn() -> get_results(year, round) end
    laps = with_cached_file "dd-#{year}-#{round}-laps.json", fn() -> get_laps(year, round) end
    pitstops = with_cached_file "dd-#{year}-#{round}-pitstops.json", fn() -> get_pitstops(year, round) end
    drivers = with_cached_file "dd-#{year}-#{round}-drivers.json", fn() -> get_drivers(year) end
    constructors = with_cached_file "dd-#{year}-constructors.json", fn() -> get_constructors(year) end

    data_filename = "dd-f1-#{year}-#{round}.json"
    chart_data = DataConverter.convert_data(results, laps, pitstops, drivers)

    qdata = get_qualy_results(year, round)

    strategy = get_strategy(drivers, pitstops, tire_data, results)

    File.write(data_filename, Poison.encode!(%{chart_data: chart_data, qdata: qdata, constructors: constructors, strategy: strategy}), [:binary])
    File.write("dd-f1-#{year}-#{round}.html", EEx.eval_file("laps.html.eex", [year: year, round: round, data_src: data_filename]))
  end


  def get_strategy(drivers, pitstops, tire_data, results) do
    tire_data 
    # create a map of 
    # %{"wehrlein": [%{"tire": "S", "from_lap": 0}, %{"tire": "M","from_lap": 11}]}
    |> Enum.reduce(%{}, fn {driver, tires}, acc -> 
      {ps, _} = Enum.map_reduce(tires, 0, fn(t, idx) -> 
        {%{from_lap: find_nth_pitstop_lap(driver, pitstops, idx), tire: t}, idx + 1}
      end)
      Map.put acc, driver, ps
    end)
    # put number of laps to last entry as :to_lap
    |> Enum.reduce(%{}, fn {driver, data}, acc -> 
      result = Enum.find(results, fn(r) -> 
        r["Driver"]["driverId"] == driver
      end)
      [last | rest] = Enum.reverse(data)
      {lap, _} = Integer.parse(result["laps"])
      last = Map.put(last, :to_lap, lap)
      data = [last | rest] |> Enum.reverse
      Map.put acc, driver, data
    end)
    # insert :to_laps for the rest of the entry
    |> Enum.reduce(%{}, fn {driver, data}, acc -> 
      {data, _} = data |> Enum.reverse |> Enum.map_reduce(0, fn(d, acc) -> 
        case Map.get d, :to_lap do
          nil -> {Map.put(d, :to_lap, acc), d.from_lap}
          _ -> {d, d.from_lap}
        end
      end)
      Map.put acc, driver, Enum.reverse(data)
    end)
    #|> IO.inspect
  end

  def find_nth_pitstop_lap(driver, pitstops, 0) do
    0
  end
  def find_nth_pitstop_lap(driver, pitstops, nth) do
    ps = Enum.find(pitstops, fn(ps) -> 
      Map.get(ps, "driverId") == driver && Map.get(ps, "stop") == "#{nth}" 
    end)
    {lap, _} = Integer.parse Map.get(ps, "lap")
    lap
  end

  def get_tire_data(year, round) do
    url = "http://www.motorsport-total.com/f1/ergeb/#{year}/#{round |> String.rjust(2, ?0)}/75ts.shtml"
    IO.inspect url
    html = get_html(url)
    table = hd(Floki.find(html, "table.table1"))
    # IO.inspect table
    {"table", _attributes, children} = table
    # IO.inspect children

    Floki.find(html, "table.table1 tr") 
    |> Enum.filter(fn(node) ->  
      case node do
        # manufacturer row
        {"tr", _, [{"td", [{"colspan", "7"} | _], _} | _]} -> false
        # header row
        {"tr", _, [{"th", _, _} | _]} -> false
        # interesting row
        {"tr", _, _} -> true
        # anything else
        _ -> false
      end
    end)
    |> Enum.reduce(%{}, fn(tr, map) ->       
      {"tr", _, tds} = tr
      [first_td | tds] = tds
      case first_td do
        {"td", _, [{"a", _, [driver_name]}]} -> 

          tires = Enum.map(tds, fn(td) -> 
            {"td", _, [x]} = td
            x = Regex.replace(~r/ ?\(DNF\)/, x, "")
            x = Map.get(%{
              "SW" => "SS",
              "W" => "S",
              "M" => "M",
              "H" => "H",<commit_msg>work around missing tire abbr.<commit_after>defmodule Data do

  @max_respone_rows 300

  def main(args) do
    args |> parse_args |> process
  end

  def process([]) do
    IO.puts "No arguments given"
  end

  def process([type: "html", year: year, round: round]) do
    data_filename = "dd-f1-#{year}-#{round}.json"
    File.write("dd-f1-#{year}-#{round}.html", EEx.eval_file("laps.html.eex", [year: year, round: round, data_src: data_filename]))
    #File.write("dd-f1-qualy-#{year}-#{round}.html", EEx.eval_file("qualy.html.eex", [year: year, round: round, data_src: data_filename]))
  end


  def process([type: "all", year: year, round: round]) do
    #get_tire_data(year, round)

    # get tire data:
    tire_data = with_cached_file "dd-#{year}-#{round}-tires.json", fn() -> get_tire_data(year, round) end

    # get race results
    results = with_cached_file "dd-#{year}-#{round}-results.json", fn() -> get_results(year, round) end
    laps = with_cached_file "dd-#{year}-#{round}-laps.json", fn() -> get_laps(year, round) end
    pitstops = with_cached_file "dd-#{year}-#{round}-pitstops.json", fn() -> get_pitstops(year, round) end
    drivers = with_cached_file "dd-#{year}-#{round}-drivers.json", fn() -> get_drivers(year) end
    constructors = with_cached_file "dd-#{year}-constructors.json", fn() -> get_constructors(year) end

    data_filename = "dd-f1-#{year}-#{round}.json"
    chart_data = DataConverter.convert_data(results, laps, pitstops, drivers)

    qdata = get_qualy_results(year, round)

    strategy = get_strategy(drivers, pitstops, tire_data, results)

    File.write(data_filename, Poison.encode!(%{chart_data: chart_data, qdata: qdata, constructors: constructors, strategy: strategy}), [:binary])
    File.write("dd-f1-#{year}-#{round}.html", EEx.eval_file("laps.html.eex", [year: year, round: round, data_src: data_filename]))
  end


  def get_strategy(drivers, pitstops, tire_data, results) do
    tire_data 
    # create a map of 
    # %{"wehrlein": [%{"tire": "S", "from_lap": 0}, %{"tire": "M","from_lap": 11}]}
    |> Enum.reduce(%{}, fn {driver, tires}, acc -> 
      {ps, _} = Enum.map_reduce(tires, 0, fn(t, idx) -> 
        {%{from_lap: find_nth_pitstop_lap(driver, pitstops, idx), tire: t}, idx + 1}
      end)
      Map.put acc, driver, ps
    end)
    # put number of laps to last entry as :to_lap
    |> Enum.reduce(%{}, fn {driver, data}, acc -> 
      result = Enum.find(results, fn(r) -> 
        r["Driver"]["driverId"] == driver
      end)
      [last | rest] = Enum.reverse(data)
      {lap, _} = Integer.parse(result["laps"])
      last = Map.put(last, :to_lap, lap)
      data = [last | rest] |> Enum.reverse
      Map.put acc, driver, data
    end)
    # insert :to_laps for the rest of the entry
    |> Enum.reduce(%{}, fn {driver, data}, acc -> 
      {data, _} = data |> Enum.reverse |> Enum.map_reduce(0, fn(d, acc) -> 
        case Map.get d, :to_lap do
          nil -> {Map.put(d, :to_lap, acc), d.from_lap}
          _ -> {d, d.from_lap}
        end
      end)
      Map.put acc, driver, Enum.reverse(data)
    end)
    #|> IO.inspect
  end

  def find_nth_pitstop_lap(driver, pitstops, 0) do
    0
  end
  def find_nth_pitstop_lap(driver, pitstops, nth) do
    ps = Enum.find(pitstops, fn(ps) -> 
      Map.get(ps, "driverId") == driver && Map.get(ps, "stop") == "#{nth}" 
    end)
    {lap, _} = Integer.parse Map.get(ps, "lap")
    lap
  end

  def get_tire_data(year, round) do
    url = "http://www.motorsport-total.com/f1/ergeb/#{year}/#{round |> String.rjust(2, ?0)}/75ts.shtml"
    IO.inspect url
    html = get_html(url)
    table = hd(Floki.find(html, "table.table1"))
    # IO.inspect table
    {"table", _attributes, children} = table
    # IO.inspect children

    Floki.find(html, "table.table1 tr") 
    |> Enum.filter(fn(node) ->  
      case node do
        # manufacturer row
        {"tr", _, [{"td", [{"colspan", "7"} | _], _} | _]} -> false
        # header row
        {"tr", _, [{"th", _, _} | _]} -> false
        # interesting row
        {"tr", _, _} -> true
        # anything else
        _ -> false
      end
    end)
    |> Enum.reduce(%{}, fn(tr, map) ->       
      {"tr", _, tds} = tr
      [first_td | tds] = tds
      case first_td do
        {"td", _, [{"a", _, [driver_name]}]} -> 
          tires = Enum.map(tds, fn(td) -> 
            # code aroucn mssinf ultra soft letter...
            x = case td do
              {"td", _, [x]} -> x  
              {"td", _, []} -> "UW"  
            end
            #{"td", _, [x]} = td
            x = Regex.replace(~r/ ?\(DNF\)/, x, "")
            x = Map.get(%{
              "UW" => "US",
              "SW" => "SS",
              "W" => "S",
              "M" => "M",
              "H" => "H",
{'fraction_non_alphanumeric': 0.18448275862068966, 'fraction_numerical': 0.005172413793103448, 'mean_word_length': 2.6169134840218238, 'pattern_counts': {'":': 10, '<': 3, '<?xml version=': 0, '>': 84, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2581441', 'n_tokens_mistral': 3470, 'n_tokens_neox': 3416, 'n_words': 765}