<commit_before>import scrapy
from scrapy.http.request import Request
from reggaeton_lyrics_scrapper.items import ReggaetonLyricsScrapperItem
import sys
reload(sys)
sys.setdefaultencoding('utf8')
class ReggaetonOnline(scrapy.Spider):
    name = 'ReggaetonOnline'
    base_url = 'http://www.reggaetonline.net/'
    start_urls = [
        'http://www.reggaetonline.net/es-artistas-reggaeton.php',
    ]
    items = []
    def parse(self, response):
        urls = response.xpath('//table[1]//tr//td/div//text()').extract()
        for url in urls:
            reggaetonLyricsScrapperItem = ReggaetonLyricsScrapperItem()
            author_name = url.lower().replace(' ', '-').replace(':','')
            reggaetonLyricsScrapperItem['author'] = author_name
            new_url = self.base_url + author_name + '_liricas-2'
            yield Request(url=new_url,meta={'item':reggaetonLyricsScrapperItem},
                callback=self.parse_songs)
        return

    def parse_songs(self,response):
        songs = response.xpath('//a[contains(@class,"NORM")][1]/text()').extract()
        if len(songs) > 1:
            for song in songs:
                song_name = song.lower().rstrip('\r\n').strip().replace(' ','-')
                if song_name:
                    reggaetonLyricsScrapperItem = response.meta['item']
                    reggaetonLyricsScrapperItem = reggaetonLyricsScrapperItem.copy()
                    reggaetonLyricsScrapperItem['name'] = song_name
                    new_url = self.base_url + song_name + '_liricas'
                    yield Request(url=new_url,meta={'item':reggaetonLyricsScrapperItem },
                    callback=self.parse_lyrics)
        return

    def parse_lyrics(self,response):
        text = response.xpath('//div[@id="artist_main"]//p[last()]/text()').extract()
        rating = response.xpath('//div[@id="artist_main"]/div[1]/div/small/span/text()').extract()
        try:
            lyric = ''.join([str(x) for x in text]).lower().rstrip('\r\n').strip()
            if lyric:
                reggaetonLyricsScrapperItem = response.meta['item']
                reggaetonLyricsScrapperItem = reggaetonLyricsScrapperItem.copy()
                reggaetonLyricsScrapperItem['lyric'] = lyric
                print '****' * 30
                print rating
                return reggaetonLyricsScrapperItem
        except Exception as e:
            print str(e)
            sys.exit()
        return None<commit_msg>feat(ReggaetonOnline): Adding proper changes to get the lyrics of the song and the autho rname<commit_after>import scrapy
from scrapy.http.request import Request
from reggaeton_lyrics_scrapper.items import ReggaetonLyricsScrapperItem
import sys
reload(sys)
sys.setdefaultencoding('utf8')
class ReggaetonOnline(scrapy.Spider):
    name = 'ReggaetonOnline'
    base_url = 'http://www.reggaetonline.net/'
    start_urls = [
        'http://www.reggaetonline.net/es-artistas-reggaeton.php',
    ]
    items = []
    def parse(self, response):
        urls = response.xpath('//table[1]//tr//td/div//text()').extract()
        for url in urls:
            reggaetonLyricsScrapperItem = ReggaetonLyricsScrapperItem()
            author_name = url.lower().replace(' ', '-').replace(':','')
            reggaetonLyricsScrapperItem['author'] = author_name
            new_url = self.base_url + author_name + '_liricas-2'
            yield Request(url=new_url,meta={'item':reggaetonLyricsScrapperItem},
                callback=self.parse_songs)
        return None

    def parse_songs(self,response):
        songs = response.xpath('//a[contains(@class,"NORM")][1]/text()').extract()
        if len(songs) > 1:
            for song in songs:
                song_name = song.lower().rstrip('\r\n').strip().replace(' ','-')
                if song_name:
                    reggaetonLyricsScrapperItem = response.meta['item']
                    reggaetonLyricsScrapperItem = reggaetonLyricsScrapperItem.copy()
                    reggaetonLyricsScrapperItem['name'] = song_name
                    new_url = self.base_url + song_name + '_liricas'
                    yield Request(url=new_url,meta={'item':reggaetonLyricsScrapperItem },
                    callback=self.parse_lyrics)
        return None

    def parse_lyrics(self,response):
        text = response.xpath('//div[@id="artist_main"]//p[last()]/text()').extract()
        lyric = ''.join([str(x) for x in text]).lower().rstrip('\r\n').strip()
        if lyric:
            reggaetonLyricsScrapperItem = response.meta['item']
            reggaetonLyricsScrapperItem = reggaetonLyricsScrapperItem.copy()
            reggaetonLyricsScrapperItem['lyric'] = lyric
            return reggaetonLyricsScrapperItem
        return None
{'fraction_non_alphanumeric': 0.1286586649821015, 'fraction_numerical': 0.0027374184038745, 'mean_word_length': 2.4977908689248896, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6534835', 'n_tokens_mistral': 1448, 'n_tokens_neox': 1443, 'n_words': 225}