<jupyter_start><jupyter_text>프로젝트 생성<jupyter_code>!scrapy startproject music<jupyter_output><empty_output><jupyter_text>items.py 작성<jupyter_code>%%writefile music/music/items.py
import scrapy

class MusicItem(scrapy.Item):
    singer = scrapy.Field()
    title = scrapy.Field()
    composer = scrapy.Field()
    lyrics = scrapy.Field()<jupyter_output>Overwriting music/music/items.py<jupyter_text>spider 작성<jupyter_code>%%writefile music/music/spiders/spider.py
import scrapy
import pandas as pd
from music.items import MusicItem

class Spider(scrapy.Spider):
    name = "MusicChart"
    allow_domain = ["music.bugs.co.kr"]
    dates = pd.date_range("20140105", "20191124", freq="7D") # 원하는 날짜 생성
    for date in dates:
        date = [str(date)[:10].replace("-", "")][0]
        start_urls = ['https://music.bugs.co.kr/chart/track/week/total?chartdate={}'.format(date)]
        
    def parse(self, response):
        links = response.xpath('//*[@id="CHARTweek"]/table/tbody/tr/td[4]/a/@href').extract()
        for link in links:
            yield scrapy.Request(link, callback=self.page_content)

    def page_content(self, response):
        item = MusicItem()
        item["title"] = response.xpath('//*[@id="container"]/header/div/h1/text()')[0].extract().replace("\r", "").replace("\n", "").replace("\t", "")
        try:
            item["singer"] = response.xpath('//*[@id="container"]/section[1]/div/div[1]/table/tbody/tr[2]/td/span[2]/a/text()')[0].extract().replace("\r", "").replace("\n", "").replace("\t", "")
        except:
            item["singer"] = response.xpath('//*[@id="container"]/section[1]/div/div[1]/table/tbody/tr[1]/td/a/text()')[0].extract().replace("\r", "").replace("\n", "").replace("\t", "")
        composer = response.xpath('//*[@id="container"]/section[1]/div/div[1]/table/tbody/tr[2]/td/span[5]/a/text()').extract()
        item["composer"] = " ".join(composer)
        try:
            item["lyrics"] = response.xpath('//*[@id="container"]/section[1]/div/div[1]/table/tbody/tr[2]/td/span[8]/a/text()')[0].extract()
        except:
            item["lyrics"] = None
        yield item<jupyter_output>Overwriting music/music/spiders/spider.py<jupyter_text>Scrapy 실행 파일 생성- csv파일도 저장<jupyter_code>%%writefile run.sh
cd music
scrapy crawl MusicChart -o MusicChart.csv
!chmod +x run.sh<jupyter_output><empty_output><jupyter_text>mongodb에 저장- 데이터 베이스 생성 파일<jupyter_code>%%writefile music/music/mongodb.py
import pymongo

client = pymongo.MongoClient('mongodb://ip:port/')
db = client.musicdb # 데이터베이스
collection = db.music # 테이블<jupyter_output>Overwriting music/music/mongodb.py<jupyter_text>- 테이블 컬럼들 설정<jupyter_code>%%writefile music/music/pipelines.py
from .mongodb import collection

class MusicPipeline(object):
    def process_item(self, item, spider):
        
        data = {
            "title" : item["title"],
            "singer" : item["singer"],
            "composer" : item["composer"],
            "lyrics" : item["lyrics"],
        }
        collection.insert(data)
        return item<jupyter_output>Overwriting music/music/pipelines.py<jupyter_text>파이프라인에 추가<jupyter_code>!echo "ITEM_PIPELINES = {" >> music/music/settings.py
!echo "    'music.pipelines.MusicPipeline': 300," >> music/music/settings.py
!echo "}" >> music/music/settings.py<jupyter_output><empty_output><jupyter_text>실행<jupyter_code>!./run.sh
!ls music
import pandas as pd
aa = pd.read_csv('/home/ubuntu/python3/notebook/music/music/MusicChart.csv')
aa.tail()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.18746438746438746, 'fraction_numerical': 0.01282051282051282, 'mean_word_length': 3.9104895104895103, 'pattern_counts': {'":': 0, '<': 29, '<?xml version=': 0, '>': 35, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '157717', 'license': 'MIT', 'n_tokens_mistral': 1359, 'n_tokens_neox': 1364, 'n_words': 175, 'path': 'music_datas.ipynb', 'repo_name': 'kimchangkyu/Music-crawling', 'size': '3510'}