<commit_before>require 'net/https'
require 'thread'
require 'nokogiri'

# Gets all the courses from the UdeM website and outputs them in HTML format

def get_next_uri
  uri = URI("https://www.etsmtl.ca/Etudiants-actuels/Baccalaureat/Cours-horaires-1er-cycle/Cours-par-titre?lettre=#{@l}")
  @l = (@l.ord + 1).chr
  return uri
end

def get_courses(uri)
  Net::HTTP.get(uri)
end

@l = 'a'
@num_of_threads = 26
@is_writing = Mutex.new

threads = []

@num_of_threads.times {
  threads << Thread.new {
    until @l.ord > 'z'.ord do
      uri = get_next_uri
      @is_writing.lock
      File.open('data.html', 'a') { |io|
        io.puts Nokogiri::HTML(
          get_courses(uri))
            .css('#plc_lt_zoneMain_pageplaceholder_pageplaceholder_lt_zoneContent_pageplaceholder_pageplaceholder_lt_zoneCenter_pageplaceholder_pageplaceholder_lt_zoneCenter_ListeCoursParTitre_GridViewResultats')
            # Why the fuck is this ID so long?!
      }
      @is_writing.unlock
    end
  }
}

threads.each { |thr| thr.join }<commit_msg>Improve char logic<commit_after>require 'net/https'
require 'thread'
require 'nokogiri'

# Gets all the courses from the UdeM website and outputs them in HTML format

def get_next_uri
  uri = URI("https://www.etsmtl.ca/Etudiants-actuels/Baccalaureat/Cours-horaires-1er-cycle/Cours-par-titre?lettre=#{@l}")
  @l = @l.next
  return uri
end

def get_courses(uri)
  Net::HTTP.get(uri)
end

@l = 'a'
@num_of_threads = 26
@is_writing = Mutex.new

threads = []

@num_of_threads.times {
  threads << Thread.new {
    until @l > 'z' do
      uri = get_next_uri
      @is_writing.lock
      File.open('data.html', 'a') { |io|
        io.puts Nokogiri::HTML(
          get_courses(uri))
            .css('#plc_lt_zoneMain_pageplaceholder_pageplaceholder_lt_zoneContent_pageplaceholder_pageplaceholder_lt_zoneCenter_pageplaceholder_pageplaceholder_lt_zoneCenter_ListeCoursParTitre_GridViewResultats')
            # Why the fuck is this ID so long?!
      }
      @is_writing.unlock
    end
  }
}

threads.each { |thr| thr.join }
{'fraction_non_alphanumeric': 0.11350293542074363, 'fraction_numerical': 0.003424657534246575, 'mean_word_length': 3.647727272727273, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 2, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1660124', 'n_tokens_mistral': 759, 'n_tokens_neox': 750, 'n_words': 143}