<jupyter_start><jupyter_text>playnomore- http://playnomore.co.kr/- scrapy에서 fake-useragent 사용- scrapy를 실행할때 아규먼트를 설정해서 실행- pipelines에서 데이터 베이스로 데이터를 저장<jupyter_code>import scrapy
import requests
from scrapy.http import TextResponse<jupyter_output><empty_output><jupyter_text>1. 프로젝트 생성<jupyter_code>!rm -rf playnomore
!scrapy startproject playnomore<jupyter_output>New Scrapy project 'playnomore', using template directory '/Users/yeonghwanchoi/opt/anaconda3/lib/python3.7/site-packages/scrapy/templates/project', created in:
    /Users/yeonghwanchoi/Documents/dev/TIL/scrapy/playnomore

You can start your first spider with:
    cd playnomore
    scrapy genspider example example.com<jupyter_text>2. items.py- title, price, img, link<jupyter_code>%%writefile playnomore/playnomore/items.py
import scrapy

class PlaynomoreItem(scrapy.Item):
    title = scrapy.Field()
    price = scrapy.Field()
    img = scrapy.Field()
    link = scrapy.Field()<jupyter_output>Overwriting playnomore/playnomore/items.py<jupyter_text>3. xpath 확인- 링크- 링크 -> 상세페이지(제목, 이미지URL, 가격)- fake_useragent 설치    - pip install fake_useragent<jupyter_code>from fake_useragent import UserAgent
url = "http://playnomore.co.kr/category/bag/24/"
# headers = { "User-Agent": UserAgent().chrome }
headers = { "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36" }
req = requests.get(url, headers=headers)
response = TextResponse(req.url, body=req.text, encoding="utf-8") 
response
# 링크
links = response.xpath(
    '//*[@id="contents"]/div[2]/div/ul/li/div/a/@href'
).extract()
links = list(map(response.urljoin, links))
# 상세페이지 : 제목, 가격, 이미지URL
url = links[0]
headers = { "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36" }
req = requests.get(url, headers=headers)
response = TextResponse(req.url, body=req.text, encoding="utf-8") 
response
title1 = response.xpath(
        '//*[@id="contents"]/div[1]/div[1]/div[2]/div[1]/font/text()'
    ).extract()
title2 = response.xpath(
        '//*[@id="contents"]/div[1]/div[1]/div[2]/div[1]/text()'
    ).extract()
title = "".join(title1) + "".join(title2)
price = response.xpath(
        '//*[@id="contents"]/div[1]/div[1]/div[2]/div[2]/text()'
    ).extract()[0]
img = "http:" + response.xpath(
        '//*[@id="contents"]/div[1]/div[1]/div[1]/div[1]/img/@src'
    ).extract()[0]
title, price, img<jupyter_output><empty_output><jupyter_text>4. spider.py- scrapy-fake-useragent 설치    - pip install scrapy-fake-useragent<jupyter_code>!pip list | grep fake
%%writefile playnomore/playnomore/spiders/spider.py
import scrapy
from playnomore.items import PlaynomoreItem

class PlaynomoreSpider(scrapy.Spider):
    name = "Playnomore"
    custom_settings = {
        'DOWNLOADER_MIDDLEWARES': {
            'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware': None,
            'scrapy_fake_useragent.middleware.RandomUserAgentMiddleware': 400,
        }
    }
    
    def start_requests(self):
        url = "http://playnomore.co.kr/category/bag/24/"
        yield scrapy.Request(url, callback=self.parse)
        
    def parse(self, response):
        links = response.xpath('//*[@id="contents"]/div[2]/div/ul/li/div/a/@href').extract()
        links = list(map(response.urljoin, links))
        for link in links:
            yield scrapy.Request(link, callback=self.page_parse)
    
    def page_parse(self, response):
        item = PlaynomoreItem()
        title1 = response.xpath('//*[@id="contents"]/div[1]/div[1]/div[2]/div[1]/font/text()').extract()
        title2 = response.xpath('//*[@id="contents"]/div[1]/div[1]/div[2]/div[1]/text()').extract()
        item["title"] = "".join(title1) + "".join(title2)
        item["price"] = response.xpath('//*[@id="contents"]/div[1]/div[1]/div[2]/div[2]/text()').extract()[0]
        item["img"] = "http:" + response.xpath('//*[@id="contents"]/div[1]/div[1]/div[1]/div[1]/img/@src').extract()[0]
        item["link"] = response.url
        yield item
%%writefile run.sh
cd playnomore
scrapy crawl Playnomore -o playnomore.csv
!chmod +x run.sh
!./run.sh
import pandas as pd
df = pd.read_csv("playnomore/playnomore.csv")
df.tail(1)<jupyter_output><empty_output><jupyter_text>5. argument 설정<jupyter_code>%%writefile playnomore/playnomore/spiders/spider.py
import scrapy
from playnomore.items import PlaynomoreItem

class PlaynomoreSpider(scrapy.Spider):
    name = "Playnomore"
    custom_settings = {
        'DOWNLOADER_MIDDLEWARES': {
            'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware': None,
            'scrapy_fake_useragent.middleware.RandomUserAgentMiddleware': 400,
        }
    }
    
    def __init__(self, category1="bag", category2=24, **kwargs):
        self.start_url = "http://playnomore.co.kr/category/{}/{}/".format(category1, category2)
        super().__init__(**kwargs)
        
    def start_requests(self):
        url = self.start_url
        yield scrapy.Request(url, callback=self.parse)
        
    def parse(self, response):
        links = response.xpath('//*[@id="contents"]/div[2]/div/ul/li/div/a/@href').extract()
        links = list(map(response.urljoin, links))
        for link in links:
            yield scrapy.Request(link, callback=self.page_parse)
    
    def page_parse(self, response):
        item = PlaynomoreItem()
        title1 = response.xpath('//*[@id="contents"]/div[1]/div[1]/div[2]/div[1]/font/text()').extract()
        title2 = response.xpath('//*[@id="contents"]/div[1]/div[1]/div[2]/div[1]/text()').extract()
        item["title"] = "".join(title1) + "".join(title2)
        item["price"] = response.xpath('//*[@id="contents"]/div[1]/div[1]/div[2]/div[2]/text()').extract()[0]
        item["img"] = "http:" + response.xpath('//*[@id="contents"]/div[1]/div[1]/div[1]/div[1]/img/@src').extract()[0]
        item["link"] = response.url
        yield item
%%writefile run.sh
cd playnomore
scrapy crawl Playnomore -o playnomore.csv -a category1=shoes -a category2=25



!./run.sh
import pandas as pd
df = pd.read_csv("playnomore/playnomore2.csv")
df.tail(1)<jupyter_output><empty_output><jupyter_text>6. Mongodb에 저장- pymongo를 pipelines.py에 적용- pip install pymongo==2.8.1<jupyter_code>!pip list | grep pymongo
import pymongo
client = pymongo.MongoClient('mongodb://13.125.153.195:27017/')
client
db = client.playnomore
collection = db.shoes
collection
data = {"title":"신발"}
collection.insert(data)<jupyter_output>/Users/yeonghwanchoi/opt/anaconda3/lib/python3.7/site-packages/ipykernel_launcher.py:2: DeprecationWarning: insert is deprecated. Use insert_one or insert_many instead.<jupyter_text>Mongodb 모듈 파일 생성<jupyter_code>%%writefile playnomore/playnomore/mongodb.py
import pymongo

client = pymongo.MongoClient('mongodb://172.16.58.3:27017/')
db = client.playnomore
collection = db.shoes
%%writefile playnomore/playnomore/pipelines.py
from .mongodb import collection

class PlaynomorePipeline(object):
    
    def process_item(self, item, spider):
        
        data = { "title": item["title"], 
                 "price": item["price"],
                 "img": item["img"], 
                 "link": item["link"],
               }
        
        collection.insert(data)
        
        return item
!echo "ITEM_PIPELINES = {"  >> playnomore/playnomore/settings.py
!echo "   'playnomore.pipelines.PlaynomorePipeline': 300," >> playnomore/playnomore/settings.py
!echo "}" >> playnomore/playnomore/settings.py
!tail -n 5 playnomore/playnomore/settings.py
!cat run.sh
!./run.sh<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16749672346002623, 'fraction_numerical': 0.025950196592398427, 'mean_word_length': 3.7933417085427137, 'pattern_counts': {'":': 8, '<': 30, '<?xml version=': 0, '>': 37, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '470908', 'license': 'MIT', 'n_tokens_mistral': 2909, 'n_tokens_neox': 2866, 'n_words': 441, 'path': 'scrapy/04_scrapy_playnomore.ipynb', 'repo_name': 'yeonghwanchoi/TIL', 'size': '7633'}