<commit_before>class SoundcloudBackend
  def self.get_url
    client = Soundcloud.new({
                                :client_id      => ENV['SOUNDCLOUD_SERVER_APP_ID'] ,
                                :client_secret  =>  ENV['SOUNDCLOUD_SERVER_APP_SECRET'],
                                :redirect_uri =>  ENV['SOUNDCLOUD_POP_REDIRECT_URL']
                            })
    puts client.authorize_url()
  end

  def self.import_url(soundcloud_url, collection_id)
    collection = Collection.find(collection_id)
    client = Soundcloud.new(:client_id =>  ENV['SOUNDCLOUD_SERVER_APP_ID'] ,:access_token => ENV['SOUNDCLOUD_POPUPARCHIVE_USER_ACCESS_TOKEN'])
    station = client.get('/resolve', :url => soundcloud_url, client_id: ENV['SOUNDCLOUD_SERVER_APP_ID'])
    tracks = client.get("/users/#{station.id}/tracks")

    tracks.each do |track|
      if track.downloadable
        item = Item.new
        item.collection = collection
        item.date_created = track.created_at
        item.title = track.title
        item.description = track.description

        instance = item.instances.build
        instance.digital    = true
        instance.format     = track.original_format
        instance.identifier = track.id

        audio = AudioFile.new
        instance.audio_files << audio
        item.audio_files << audio
        audio.identifier        =  track.download_url+"?client_id="+ENV['SOUNDCLOUD_SERVER_APP_ID']
        audio.remote_file_url   =  track.download_url+"?client_id="+ENV['SOUNDCLOUD_SERVER_APP_ID']
        item.save!
      end
    end
  end

end<commit_msg>cleanup sc backend, prevent dupes<commit_after>class SoundcloudBackend

  def self.get_url
    client = soundcloud_client
    puts client.authorize_url()
  end

  def self.import_url(soundcloud_url, collection_id)
    collection = Collection.find(collection_id)
    client     = soundcloud_client
    station    = client.get('/resolve', :url => soundcloud_url, client_id: ENV['SOUNDCLOUD_SERVER_APP_ID'])
    tracks     = client.get("/users/#{station.id}/tracks")

    tracks.each do |track|
      if track.downloadable
        
        identifier = track.uri
        next if Item.where(identifier: identifier, collection_id: collection.id).exists?

        item = Item.new
        item.identifier   = identifier
        item.collection   = collection
        item.date_created = track.created_at
        item.title        = track.title
        item.description  = track.description

        audio = AudioFile.new
        audio.identifier      = track.download_url+"?client_id="+ENV['SOUNDCLOUD_SERVER_APP_ID']
        audio.remote_file_url = track.download_url+"?client_id="+ENV['SOUNDCLOUD_SERVER_APP_ID']
        item.audio_files << audio

        item.save!
      end
    end
  end

  def self.soundcloud_client
    Soundcloud.new({
      :client_id     => ENV['SOUNDCLOUD_SERVER_APP_ID'] ,
      :client_secret => ENV['SOUNDCLOUD_SERVER_APP_SECRET'],
      :redirect_uri  => ENV['SOUNDCLOUD_POP_REDIRECT_URL']
    })
  end

end
{'fraction_non_alphanumeric': 0.09827928524156188, 'fraction_numerical': 0.0, 'mean_word_length': 2.6334134615384617, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3679306', 'n_tokens_mistral': 938, 'n_tokens_neox': 929, 'n_words': 146}