<commit_before>module Kookie
  module PureFTPD

    class LogParser

      def initialize(log_lines)
        @log_lines = log_lines
      end

      def parse
        @parsed_lines ||= parse_lines
      end

      protected

        def parse_lines
          formatted_lines = []
          @log_lines.each do |line|

            columns = find_columns(line)

            formatted_lines << {
              ip: columns[0],
              username: ensure_safe_encoding(columns[1]),
              date: parse_date(columns[2]),
              method: columns[3],
              path: URI.unescape(ensure_safe_encoding(columns[4])),
              status_code: columns[5].to_i,
              bytes: columns[6].to_i
            }
          end

          formatted_lines
        end

        def parse_date(date_string)
          Time.strptime(date_string, '%d/%b/%Y:%T %z')
        end

        def find_columns(line)
          /(.*) - (.*) \[(.*)\] "(.*) (.*)" (.*) (.*)/.match(line).captures

          # A log entry will look something like this
          # 172.16.31.10 - gyj [13/Aug/2014:12:09:22 -0000] \"PUT /home/ftp-accounts/gyj/miabelleza/MIA%20BELLEZA%20108.PDF\" 200 17233868

          # And this is how it's broken down using the pattern above
          # columns[0] = 172.16.31.10
          # columns[1] = gyj
          # columns[2] = 13/Aug/2014:12:09:22 -0000
          # columns[3] = PUT
          # columns[4] = /home/ftp-accounts/gyj/miabelleza/MIA%20BELLEZA%20108.PDF
          # columns[5] = 200
          # columns[6] = 17233868
        end

        def ensure_safe_encoding(text)
          text.force_encoding('UTF-8')
        end
    end
  end
end<commit_msg>Fixes utf-8 errors with logs<commit_after>module Kookie
  module PureFTPD

    class LogParser

      def initialize(log_lines)
        @log_lines = log_lines
      end

      def parse
        @parsed_lines ||= parse_lines
      end

      protected

        def parse_lines
          formatted_lines = []
          @log_lines.each do |line|

            columns = find_columns(line)

            formatted_lines << {
              ip: columns[0],
              username: ensure_safe_encoding(columns[1]),
              date: parse_date(columns[2]),
              method: columns[3],
              path: ensure_safe_encoding(URI.unescape(columns[4])),
              status_code: columns[5].to_i,
              bytes: columns[6].to_i
            }
          end

          formatted_lines
        end

        def parse_date(date_string)
          Time.strptime(date_string, '%d/%b/%Y:%T %z')
        end

        def find_columns(line)
          /(.*) - (.*) \[(.*)\] "(.*) (.*)" (.*) (.*)/.match(line).captures

          # A log entry will look something like this
          # 172.16.31.10 - gyj [13/Aug/2014:12:09:22 -0000] \"PUT /home/ftp-accounts/gyj/miabelleza/MIA%20BELLEZA%20108.PDF\" 200 17233868

          # And this is how it's broken down using the pattern above
          # columns[0] = 172.16.31.10
          # columns[1] = gyj
          # columns[2] = 13/Aug/2014:12:09:22 -0000
          # columns[3] = PUT
          # columns[4] = /home/ftp-accounts/gyj/miabelleza/MIA%20BELLEZA%20108.PDF
          # columns[5] = 200
          # columns[6] = 17233868
        end

        def ensure_safe_encoding(text)
          text.encode!(Encoding::ASCII).encode('UTF-8', 'US-ASCII', invalid: :replace, undef: :replace)
        end
    end
  end
end
{'fraction_non_alphanumeric': 0.12441724941724942, 'fraction_numerical': 0.05914918414918415, 'mean_word_length': 1.7181314330958037, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 4, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2621559', 'n_tokens_mistral': 1254, 'n_tokens_neox': 1090, 'n_words': 212}