<commit_before>#!/usr/bin/env python
# -*- coding=UTF-8 -*-
# Created at May 31 08:37 by <EMAIL>

import sys
if hasattr(sys, 'setdefaultencoding'):
	sys.setdefaultencoding('UTF-8')

from BeautifulSoup import Tag
from url_parser import geturl
import os
import urllib2
from mutagen.easyid3 import EasyID3
from mutagen.id3 import ID3, APIC
class track:
	def __init__(self, tag):
		self.id3 = dict()
		for i in ('title', 'album_name', 'artist'):
			self.id3[i] = tag.find(name=i).text
			self.id3[i] = self.id3[i] #.partition(r'<![CDATA[ ')[2]
			self.id3[i] = self.id3[i] #.rpartition(r']]')[0]
		self.url = geturl(tag.find(name='location').text.strip())
		self.picurl = tag.find(name='pic').text.strip()
		self.lyricurl = tag.find(name='lyric').text.strip()
		self._file = self.id3['artist'] + ' - ' + \
				self.id3['title'] + '.' + self.url.rpartition('.')[2]
		sys.stderr.write(self._file + '\n')
	def download(self, path, ReportHook = None):
		self.filename = os.path.join(path, self._file)
		self.lyricfilename = os.path.join(path, 'lyrics', self._file.rpartition('.')[0] + '.lrc')
		if os.path.exists(self.filename):
			sys.stderr.write(' File exists. Ignore.')
			return
		req = urllib2.urlopen(self.url)
		total_size = int(req.info().getheader('Content-Length').strip())
		fout = open(self.filename, 'wb')
		
		read_size = 0
		trunk_size = 10240
		try:
			while True:
				_little_data = req.read(trunk_size)
				read_size += len(_little_data)
				if len(_little_data) == 0:
					break
				fout.write(_little_data)
				if ReportHook != None:
					ReportHook(read_size, total_size)
			fout.close()
		except KeyboardInterrupt:
			fout.close()
			os.remove(self.filename)
			sys.stderr.write('\n\nDownload cancelled. ' + self._file + ' removed. Exit.\n')
			sys.exit(0)

		easyid3 = EasyID3()
		easyid3['title'] = self.id3['title']
		easyid3['album'] = self.id3['album_name']
		easyid3['artist'] = self.id3['artist']
		easyid3.save(self.filename)

		pic = urllib2.urlopen(self.picurl)
		harid3 = ID3(self.filename)
		harid3.add(APIC(\
				encoding = 3, \
				mime = pic.info().type, \
				type = 3, \
				desc = 'Cover picture from xiami.com fetched by BlahGeek.', \
				data = pic.read()))
		harid3.save()
		sys.stderr.write(' ID3 OK.')

		fout = open(self.lyricfilename, 'wb')
		fout.write(urllib2.urlopen(self.lyricurl).read())
		fout.close()
		sys.stderr.write(' Lyrics OK.')<commit_msg>performer<commit_after>#!/usr/bin/env python
# -*- coding=UTF-8 -*-
# Created at May 31 08:37 by <EMAIL>

import sys
if hasattr(sys, 'setdefaultencoding'):
	sys.setdefaultencoding('UTF-8')

from BeautifulSoup import Tag
from url_parser import geturl
import os
import urllib2
from mutagen.easyid3 import EasyID3
from mutagen.id3 import ID3, APIC
class track:
	def __init__(self, tag):
		self.id3 = dict()
		for i in ('title', 'album_name', 'artist'):
			self.id3[i] = tag.find(name=i).text
			self.id3[i] = self.id3[i] #.partition(r'<![CDATA[ ')[2]
			self.id3[i] = self.id3[i] #.rpartition(r']]')[0]
		self.url = geturl(tag.find(name='location').text.strip())
		self.picurl = tag.find(name='pic').text.strip()
		self.lyricurl = tag.find(name='lyric').text.strip()
		if not self.lyricurl.endswith('.lrc'):
			self.lyricurl = None
		self._file = self.id3['artist'] + ' - ' + \
				self.id3['title'] + '.' + self.url.rpartition('.')[2]
		sys.stderr.write(self._file + '\n')
	def download(self, path, ReportHook = None):
		self.filename = os.path.join(path, self._file)
		self.lyricfilename = os.path.join(path, 'lyrics', self._file.rpartition('.')[0] + '.lrc')
		if os.path.exists(self.filename):
			sys.stderr.write(' File exists. Ignore.')
			return
		req = urllib2.urlopen(self.url)
		total_size = int(req.info().getheader('Content-Length').strip())
		fout = open(self.filename, 'wb')
		
		read_size = 0
		trunk_size = 10240
		try:
			while True:
				_little_data = req.read(trunk_size)
				read_size += len(_little_data)
				if len(_little_data) == 0:
					break
				fout.write(_little_data)
				if ReportHook != None:
					ReportHook(read_size, total_size)
			fout.close()
		except KeyboardInterrupt:
			fout.close()
			os.remove(self.filename)
			sys.stderr.write('\n\nDownload cancelled. ' + self._file + ' removed. Exit.\n')
			sys.exit(0)

		easyid3 = EasyID3()
		easyid3['title'] = self.id3['title']
		easyid3['album'] = self.id3['album_name']
		easyid3['artist'] = self.id3['artist'].split(';')
		easyid3['performer'] = easyid3['artist'][0]
		easyid3.save(self.filename)

		pic = urllib2.urlopen(self.picurl)
		harid3 = ID3(self.filename)
		harid3.add(APIC(\
				encoding = 3, \
				mime = pic.info().type, \
				type = 3, \
				desc = 'Cover picture from xiami.com fetched by BlahGeek.', \
				data = pic.read()))
		harid3.save()
		sys.stderr.write(' ID3 OK.')

		if self.lyricurl != None:
			fout = open(self.lyricfilename, 'wb')
			fout.write(urllib2.urlopen(self.lyricurl).read())
			fout.close()
			sys.stderr.write(' Lyrics OK.')
		else:
			sys.stderr.write(' Lyrics not available.')
{'fraction_non_alphanumeric': 0.17935871743486975, 'fraction_numerical': 0.021442885771543085, 'mean_word_length': 3.717391304347826, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5460451', 'n_tokens_mistral': 2193, 'n_tokens_neox': 1922, 'n_words': 358}