<commit_before>class ScanSources
  def process
    sources = Source.all
    sources.each do | source |
      proces_source
    end
  end

  private

  def process_source
    ['mp4','mp3'].each do | ext |
      process_extention
    end
  end

  def process_extention
    pattern = File.join(source.path, '*.'+ext)
    list = Dir.glob(pattern)
    list.each do | filename |
      process_file
    end
  end

  def process_file
    base = File.basename(filename)
    medium = Medium.new
    medium.filename = base
    medium.save
  end
end<commit_msg>refactor ScanSources process method, add method params<commit_after>class ScanSources
  def process
    sources = Source.all
    sources.each do | source |
      proces_source(source)
    end
  end

  private

  def process_source(source)
    ['mp4','mp3'].each do | ext |
      process_extention(ext)
    end
  end

  def process_extention(ext)
    pattern = File.join(source.path, '*.'+ext)
    list = Dir.glob(pattern)
    list.each do | filename |
      process_file(filename)
    end
  end

  def process_file(filename)
    base = File.basename(filename)
    medium = Medium.new
    medium.filename = base
    medium.save
  end
end
{'fraction_non_alphanumeric': 0.08691983122362869, 'fraction_numerical': 0.0033755274261603376, 'mean_word_length': 2.6269113149847096, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1514575', 'n_tokens_mistral': 387, 'n_tokens_neox': 373, 'n_words': 95}