<commit_before>#!/usr/bin/env ruby
# encoding: utf-8

require 'json'
require 'set'
require 'digest/md5'
require 'csv'

module Ebooks
  class Model
    attr_accessor :hash, :sentences, :mentions, :keywords

    def self.consume(txtpath)
      Model.new.consume(txtpath)
    end

    def self.load(path)
      Marshal.load(File.open(path, 'rb') { |f| f.read })
    end

    def consume(path)
      content = File.read(path)
      @hash = Digest::MD5.hexdigest(content)

      if path.split('.')[-1] == "json"
        log "Reading json corpus from #{path}"
        lines = JSON.parse(content, symbolize_names: true).map do |tweet|
          tweet[:text]
        end
      elsif path.split('.')[-1] == "csv"
        log "Reading CSV corpus from #{path}"
        header = CSV.read(path).first
        text_col = header.index('text')
        lines = CSV.read(path).drop(1).map do |tweet|
          tweet[text_col]
        end
      else
        log "Reading plaintext corpus from #{path}"
        lines = content.split("\n")
      end

      log "Removing commented lines and sorting mentions"

      keeping = []
      mentions = []
      lines.each do |l|
        next if l.start_with?('#') # Remove commented lines
        next if l.include?('RT') || l.include?('MT') # Remove soft retweets
        
        if l.include?('@')
          mentions << l
        else
          keeping << l
        end
      end
      text = NLP.normalize(keeping.join("\n")) # Normalize weird characters
      mention_text = NLP.normalize(mentions.join("\n"))

      log "Segmenting text into sentences"

      statements = NLP.sentences(text)
      mentions = NLP.sentences(mention_text)

      log "Tokenizing #{statements.length} statements and #{mentions.length} mentions"
      @sentences = []<commit_msg>MODEL: Read in utf-8, only parse CSV once<commit_after>#!/usr/bin/env ruby
# encoding: utf-8

require 'json'
require 'set'
require 'digest/md5'
require 'csv'

module Ebooks
  class Model
    attr_accessor :hash, :sentences, :mentions, :keywords

    def self.consume(txtpath)
      Model.new.consume(txtpath)
    end

    def self.load(path)
      Marshal.load(File.open(path, 'rb') { |f| f.read })
    end

    def consume(path)
      content = File.read(path, :encoding => 'utf-8')
      @hash = Digest::MD5.hexdigest(content)

      if path.split('.')[-1] == "json"
        log "Reading json corpus from #{path}"
        lines = JSON.parse(content, symbolize_names: true).map do |tweet|
          tweet[:text]
        end
      elsif path.split('.')[-1] == "csv"
        log "Reading CSV corpus from #{path}"
        content = CSV.parse(content)
        header = content.shift
        text_col = header.index('text')
        lines = content.map do |tweet|
          tweet[text_col]
        end
      else
        log "Reading plaintext corpus from #{path}"
        lines = content.split("\n")
      end

      log "Removing commented lines and sorting mentions"

      keeping = []
      mentions = []
      lines.each do |l|
        next if l.start_with?('#') # Remove commented lines
        next if l.include?('RT') || l.include?('MT') # Remove soft retweets
        
        if l.include?('@')
          mentions << l
        else
          keeping << l
        end
      end
      text = NLP.normalize(keeping.join("\n")) # Normalize weird characters
      mention_text = NLP.normalize(mentions.join("\n"))

      log "Segmenting text into sentences"

      statements = NLP.sentences(text)
      mentions = NLP.sentences(mention_text)

      log "Tokenizing #{statements.length} statements and #{mentions.length} mentions"
      @sentences = []
{'fraction_non_alphanumeric': 0.13155726448777808, 'fraction_numerical': 0.0035704476792090085, 'mean_word_length': 2.304900181488203, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7362111', 'n_tokens_mistral': 1171, 'n_tokens_neox': 1122, 'n_words': 295}