<commit_before># encoding: utf-8

require_relative 'location_decoder.rb'

class XiamiCloner

	require 'fileutils'
	require 'digest/md5'
	require 'nokogiri'
	require 'net/http'
	require 'ruby-pinyin'
	require 'image_science'

	INFO_URL = 'http://www.xiami.com/song/playlist/id/%d/object_name/default/object_id/0'
	ALBUM_PAGE_URL = 'http://www.xiami.com/album/%d'
	CACHE_DIR = '/tmp/xiami_cloner_cache'

	def self.clone(playlist, outdir, options = {})
		cloneds = options[:cloned_file] && File.exists?(options[:cloned_file])?
			strip_invalid(File.read(options[:cloned_file]).lines) :
			[]
		songs = strip_invalid(File.read(playlist).lines)

		counter = 0

		songs.each do |song|
			counter += 1

			print "正在下载第 #{counter} / #{songs.size} 首歌曲"

			if cloneds.include?(song)
				puts " ... 跳过"
				next
			end

			puts

			self.clone_song(song, outdir, options)

			cloneds << song
			File.write(options[:cloned_file], cloneds.join("\n")) if options[:cloned_file]
		end
	end

	def self.check_integrity(playlist)
		songs = strip_invalid(File.read(playlist).lines)

		counter = 0

		songs.each do |song|
			counter += 1

			puts "正在检查第 #{counter} / #{songs.size} 首歌曲的完整性"

			puts "  歌曲 #{song} 没有下载完全" unless check_song_integrity(song)
		end
	end

	private
		def self.check_song_integrity(song)
			return true if File.exists?(cache_path("#{song}.complete"))

			info = retrieve_info(song)
			url = LocationDecoder.decode(info.search('location').text)

			size = get_content_size(url)
			download_to_cache(url, "#{song}.mp3", false)

			if File.size(cache_path("#{song}.mp3")) == size
				FileUtils.touch(cache_path("#{song}.complete"))
				return true
			else
				return false
			end
		end

		def self.retrieve_info(id)
			info_url = INFO_URL % id
			info_path = "#{id}.info"

			if File.exists?(cache_path(info_path)) && (File.ctime(cache_path(info_path)) < (Time.now - 3600))
				# Remove if cached more than an hour ago
				FileUtils.rm(cache_path(info_path))
			end

			self.download_to_cache(info_url, info_path)

			Nokogiri::XML(File.read(self.cache_path(info_path)))
		end

		def self.retrieve_order(page, id)
			node = page.at_css('#track .chapter')
			
			node.css('.track_list').each_with_index do |d, i|
				disc = i + 1
				d.css('tr td.song_name a').each_with_index do |s, i|
					song = i + 1
					return [disc.to_s, song.to_s] if s['href'].include?(id.to_s)
				end
			end

			['', '']
		end

		def self.retrieve_publish_year(page)
			node = page.at_css('#album_block table')

			node.css('tr').each do |r|
				ds = r.css('td')
				if ds[0].text == '发行时间：'
					match = /([0-9]*)年/.match(ds[1].text)
					if match
						return match[1]
					else
						return ''
					end
				end
			end

			''
		end

		def self.retrieve_album_page(id)
			page_url = ALBUM_PAGE_URL % id
			page_path = "album_#{id}.page"

			self.download_to_cache(page_url, page_path)

			Nokogiri::HTML(File.read(self.cache_path(page_path)))
		end

		def self.strip_invalid(list)
			list.select { |x| x.to_i > 0 }.map { |x| x.strip }
		end

		def self.clone_song(song, outdir, options = {})
			options[:import_to_itunes] ||= false

			FileUtils.mkdir_p outdir

			info = retrieve_info(song)
			url = LocationDecoder.decode(info.search('location').text)

			while true
				break if check_song_integrity(song)
				FileUtils.rm(self.cache_path("#{song}.mp3"))
				self.download_to_cache(url, "#{song}.mp3", false)
			end

			out_path = File.join(outdir, filename(song))
			out_path = uniquefy(out_path, ".mp3")

			FileUtils.cp(self.cache_path("#{song}.mp3"), out_path)

			write_id3(song, out_path)

			import_to_itunes(out_path) if options[:import_to_itunes]
		end

		def self.filename(song)
			info = retrieve_info(song)
			"#{info.search('artist').text} - #{info.search('title').text}"
		end

		def self.import_to_itunes(file)
		    itd = File.expand_path("~/Music/iTunes/iTunes Media/Automatically Add to iTunes.localized") 

		    FileUtils.cp(file, itd)
		end

		def self.uniquefy(filename, extname)
			return filename + extname unless File.exists?(filename + extname)

			i = 2

			while true
				new_path = filename + " " + i.to_s + extname
				return new_path unless File.exists?(new_path)
				i += 1
			end
		end

		def self.download_to_cache(url, filename, hidden = true)
		    require 'fileutils'

		    FileUtils.mkdir_p(File.expand_path(CACHE_DIR))

		    ccp = File.join(File.expand_path(CACHE_DIR), filename + ".tmp")
		    cfp = File.join(File.expand_path(CACHE_DIR), filename)

		    if !File.exists?(cfp)
		    	FileUtils.rm_rf(ccp)
		    	command = "curl --connect-timeout 15 --retry 999 --retry-max-time 0 -C - -# \"#{url}\" -o \"#{ccp}\""
		    	command += " > /dev/null 2>&1" if hidden
		    	system(command)
		    	FileUtils.mv(ccp, cfp)
		    end
		end
<commit_msg>Fixes the problem that some invalid Xiami links link to 404.<commit_after># encoding: utf-8

require_relative 'location_decoder.rb'

class XiamiCloner

	require 'fileutils'
	require 'digest/md5'
	require 'nokogiri'
	require 'net/http'
	require 'ruby-pinyin'
	require 'image_science'

	INFO_URL = 'http://www.xiami.com/song/playlist/id/%d/object_name/default/object_id/0'
	ALBUM_PAGE_URL = 'http://www.xiami.com/album/%d'
	CACHE_DIR = '/tmp/xiami_cloner_cache'

	def self.clone(playlist, outdir, options = {})
		cloneds = options[:cloned_file] && File.exists?(options[:cloned_file])?
			strip_invalid(File.read(options[:cloned_file]).lines) :
			[]
		songs = strip_invalid(File.read(playlist).lines)

		counter = 0

		songs.each do |song|
			counter += 1

			print "正在下载第 #{counter} / #{songs.size} 首歌曲"

			if cloneds.include?(song)
				puts " ... 跳过"
				next
			end

			puts

			self.clone_song(song, outdir, options)

			cloneds << song
			File.write(options[:cloned_file], cloneds.join("\n")) if options[:cloned_file]
		end
	end

	def self.check_integrity(playlist)
		songs = strip_invalid(File.read(playlist).lines)

		counter = 0

		songs.each do |song|
			counter += 1

			puts "正在检查第 #{counter} / #{songs.size} 首歌曲的完整性"

			puts "  歌曲 #{song} 没有下载完全" unless check_song_integrity(song)
		end
	end

	private
		def self.check_song_integrity(song)
			return true if File.exists?(cache_path("#{song}.complete"))

			info = retrieve_info(song)
			url = LocationDecoder.decode(info.search('location').text)

			size = get_content_size(url)
			download_to_cache(url, "#{song}.mp3", false)

			if File.size(cache_path("#{song}.mp3")) == size
				FileUtils.touch(cache_path("#{song}.complete"))
				return true
			else
				return false
			end
		end

		def self.retrieve_info(id)
			info_url = INFO_URL % id
			info_path = "#{id}.info"

			if File.exists?(cache_path(info_path)) && (File.ctime(cache_path(info_path)) < (Time.now - 3600))
				# Remove if cached more than an hour ago
				FileUtils.rm(cache_path(info_path))
			end

			self.download_to_cache(info_url, info_path)

			Nokogiri::XML(File.read(self.cache_path(info_path)))
		end

		def self.retrieve_order(page, id)
			node = page.at_css('#track .chapter')
			
			node.css('.track_list').each_with_index do |d, i|
				disc = i + 1
				d.css('tr td.song_name a').each_with_index do |s, i|
					song = i + 1
					return [disc.to_s, song.to_s] if s['href'].include?(id.to_s)
				end
			end

			['', '']
		end

		def self.retrieve_publish_year(page)
			node = page.at_css('#album_block table')

			node.css('tr').each do |r|
				ds = r.css('td')
				if ds[0].text == '发行时间：'
					match = /([0-9]*)年/.match(ds[1].text)
					if match
						return match[1]
					else
						return ''
					end
				end
			end

			''
		end

		def self.retrieve_album_page(id)
			page_url = ALBUM_PAGE_URL % id
			page_path = "album_#{id}.page"

			self.download_to_cache(page_url, page_path)

			Nokogiri::HTML(File.read(self.cache_path(page_path)))
		end

		def self.strip_invalid(list)
			list.select { |x| x.to_i > 0 }.map { |x| x.strip }
		end

		def self.clone_song(song, outdir, options = {})
			options[:import_to_itunes] ||= false

			FileUtils.mkdir_p outdir

			info = retrieve_info(song)
			url = LocationDecoder.decode(info.search('location').text)

			while true
				break if check_song_integrity(song)
				FileUtils.rm(self.cache_path("#{song}.mp3"))
				self.download_to_cache(url, "#{song}.mp3", false)
			end

			out_path = File.join(outdir, filename(song))
			out_path = uniquefy(out_path, ".mp3")

			FileUtils.cp(self.cache_path("#{song}.mp3"), out_path)

			write_id3(song, out_path)

			import_to_itunes(out_path) if options[:import_to_itunes]
		end

		def self.filename(song)
			info = retrieve_info(song)
			"#{info.search('artist').text} - #{info.search('title').text}"
		end

		def self.import_to_itunes(file)
		    itd = File.expand_path("~/Music/iTunes/iTunes Media/Automatically Add to iTunes.localized") 

		    FileUtils.cp(file, itd)
		end

		def self.uniquefy(filename, extname)
			return filename + extname unless File.exists?(filename + extname)

			i = 2

			while true
				new_path = filename + " " + i.to_s + extname
				return new_path unless File.exists?(new_path)
				i += 1
			end
		end

		def self.download_to_cache(url, filename, hidden = true)
		    require 'fileutils'

		    FileUtils.mkdir_p(File.expand_path(CACHE_DIR))

		    ccp = File.join(File.expand_path(CACHE_DIR), filename + ".tmp")
		    cfp = File.join(File.expand_path(CACHE_DIR), filename)

		    if !File.exists?(cfp)
		    	FileUtils.rm_rf(ccp)
		    	command = "curl --connect-timeout 15 --retry 999 --retry-max-time 0 -C - -# \"#{url}\" -o \"#{ccp}\""
		    	command += " > /dev/null 2>&1" if hidden
		    	system(command)

		    	if !File.exists?(ccp)
		    		# TODO FIXME
		    		# If curl goes to 404 or other errors, create stub file
		    		FileUtils.touch(cfp)
		    	else
			    	FileUtils.mv(ccp, cfp)
			    end
		    end
		end

{'fraction_non_alphanumeric': 0.1435455757202912, 'fraction_numerical': 0.007997539218701938, 'mean_word_length': 3.3102076889085286, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 22, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1405924', 'n_tokens_mistral': 4413, 'n_tokens_neox': 3691, 'n_words': 702}