<jupyter_start><jupyter_text>롸켓펀치 크롤링<jupyter_code>import scrapy
from scrapy.http import TextResponse
# __init__
# start url
url = "https://www.rocketpunch.com/api/jobs/template?&page=1&q=&specialty=Python&tag=%EB%8D%B0%EC%9D%B4%ED%84%B0"

# get_pages 함수
req = requests.get(url)
response = TextResponse(req.url, body=req.text, encoding='utf-8')
response<jupyter_output><empty_output><jupyter_text>company_name = scrapy.Field()    business = scrapy.Field()    position = scrapy.Field()    link = scrapy.Field()    salary_condition = scrapy.Field()    deadline = scrapy.Field()    keyword = scrapy.Field()    location = scrapy.Field()<jupyter_code># company_name
response.text
# business
response.xpath('//*')
len(dom.select('#company-list > div.company.item > div.content > div.company-jobs-detail > div:nth-child(1) > div:nth-child(1) > a'))
len(dom.select('#company-list > div.company.item > div.content > div.company-jobs-detail > div.job-detail > div:nth-child(1) > a'))
dom.select('#company-list > div.company.item > div.content')[2].select('div.company-name > a > h4 > strong')[0].text
dom.select('#company-list > div.company.item > div.content')[1].select('div.description')[0].text.strip('\xa0')
def rocketpunch_crawler2(pages):
    datas = []
    for page in range(1,pages+1):
        print(page)
        url = "https://www.rocketpunch.com/api/jobs/template?page={}&q=&specialty=Python&tag=%EB%8D%B0%EC%9D%B4%ED%84%B0%3E".format(page)
        response = requests.get(url)
        data = response.json()['data']['template']
        dom = BeautifulSoup(data, "html.parser")
        root = dom.select('#company-list > div.company.item > div.content')
        for i in range(20):
            datas.append({
                "company" : root[i].select('div.company-name > a > h4 > strong')[0].text,
                "description" : root[i].select('div.description')[0].text.strip('\xa0'),
                "service" : root[i].select('div.nowrap.meta')[0].text.strip(),
                "position" : root[i].select('div.company-jobs-detail > div:nth-child(1) > div:nth-child(1) > a')[0].text,
                "link" : "https://www.rocketpunch.com"+ root[i].select('div.company-jobs-detail > div:nth-child(1) > div:nth-child(1) > a')[0].get('href'),
                "salary" : root[i].select('div.company-jobs-detail > div:nth-child(1) > div:nth-child(1) > span')[0].text,
                "deadline" : root[i].select('div.company-jobs-detail > div.job-detail > div.job-dates > span:nth-child(1)')[0].text.strip(' \n'),
            })
            
    return pd.DataFrame(datas)
def rocketpunch_crawler(pages):
    datas = []
    for page in range(1,pages+1):
        print(page)
        url = "https://www.rocketpunch.com/api/jobs/template?page={}&q=&specialty=Python&tag=%EB%8D%B0%EC%9D%B4%ED%84%B0%3E".format(page)
        response = requests.get(url)
        data = response.json()['data']['template']
        dom = BeautifulSoup(data, "html.parser")
        for i in range(20):
            datas.append({
                "company" : dom.select('#company-list > div.company.item > div.content > div.company-name > a > h4 > strong')[i].text,
                "description" : dom.select('#company-list > div.company.item > div.content > div.description')[i].text.strip('\xa0'),
                "service" : dom.select('#company-list > div.company.item > div.content > div.nowrap.meta')[i].text.strip(),
                "position" : dom.select('#company-list > div.company.item > div.content > div.company-jobs-detail > div:nth-child(1) > div:nth-child(1) > a')[i].text,
                "link" : "https://www.rocketpunch.com"+ dom.select('#company-list > div.company.item > div.content > div.company-jobs-detail > div:nth-child(1) > div:nth-child(1) > a')[0].get('href'),
                "salary" : dom.select('#company-list > div.company.item > div.content > div.company-jobs-detail > div:nth-child(1) > div:nth-child(1) > span')[i].text,
                "deadline" : dom.select('#company-list > div.company.item > div.content > div.company-jobs-detail > div.job-detail > div.job-dates > span:nth-child(1)')[i].text.strip(' \n'),
            })
            
    return pd.DataFrame(datas)
url = "https://www.rocketpunch.com/jobs?specialty=Python&tag=%EB%8D%B0%EC%9D%B4%ED%84%B0"
response = requests.get(url)
response
response.text
# json으로 파싱후, data안 template 안에 들어있는 애들을 BrautifulSoup으로 html로 파싱
data = response.json()['data']['template']
dom = BeautifulSoup(data, "html.parser")
root = dom.select('#company-list > div.company.item > div.content')
# 회사명
dom.select('#company-list > div.company.item > div.content > div.company-name > a > h4 > strong')[0].text
root[0].select('div.company-name > a > h4 > strong')[0].text
# 회사 설명
dom.select('#company-list > div.company.item > div.content > div.description')[0].text.strip('\xa0')
root[0].select('div.description')[0].text.strip('\xa0')
# 분야
dom.select('#company-list > div.company.item > div.content > div.nowrap.meta')[0].text.strip()
root[0].select('div.nowrap.meta')[0].text.strip()
len(root)
# 중복 있음
positions = dom.select('#company-list > div.company.item > div.content > div.company-jobs-detail')
len_of_positions = len(positions[0].select('div:nth-child(1) > a.nowrap'))
len_of_positions
# position
positions[3].select('div:nth-child(1) > a.nowrap')[0].text
# 링크
"https://www.rocketpunch.com" + positions[0].select('div:nth-child(1) > a.nowrap')[0].get('href')
# 연봉 / 스톡옵션 / 경력 여부
positions[0].select('div.job-detail > div:nth-child(1) > span')[0].text
# 채용 기간
positions[0].select('div.job-detail > div.job-dates > span:nth-child(1)')[0].text.strip(' \n')
list(range(len_of_positions))
positions[18].select('div:nth-child(1) > a.nowrap')[0].text
root[4].select('div.company-name > a > h4 > strong')[0].text
positions[4].select('div:nth-child(1) > a.nowrap')[0].text
len_of_positions = len(positions[1].select('div:nth-child(1) > a.nowrap'))
list(range(len_of_positions))
root[1].select('div.company-name > a > h4 > strong')[0].text,
url = "https://www.rocketpunch.com/api/jobs/template?&page=1&q=&specialty=Python&tag=%EB%8D%B0%EC%9D%B4%ED%84%B0"
response = requests.get(url)
data = response.json()['data']['template']
dom = BeautifulSoup(data, "html.parser")
total_pages = dom.select("div.ui.pagination.menu > div.tablet.computer.large.screen.widescreen.only > a")[-1].text
total_pages
# 크롤링 완성
import time
from datetime import date, timedelta
# 전체 페이지의 개수를 알아내기 위한 과정
url = "https://www.rocketpunch.com/api/jobs/template?&page=1&q=&specialty=Python&tag=%EB%8D%B0%EC%9D%B4%ED%84%B0"
response = requests.get(url)
data = response.json()['data']['template']
dom = BeautifulSoup(data, "html.parser")
total_pages = dom.select("div.ui.pagination.menu > div.tablet.computer.large.screen.widescreen.only > a")[-1].text

datas = []
for page in range(1,int(total_pages)+1):
    url = "https://www.rocketpunch.com/api/jobs/template?page={}&q=&specialty=Python&tag=%EB%8D%B0%EC%9D%B4%ED%84%B0%".format(page)
    response = requests.get(url)
    data = response.json()['data']['template']
    dom = BeautifulSoup(data, "html.parser")
    root = dom.select('#company-list > div.company.item > div.content')
    positions = dom.select('#company-list > div.company.item > div.content > div.company-jobs-detail')
    a = 0
    for i in range(len(root)):
        len_of_positions = len(positions[i].select('div:nth-child(1) > a.nowrap'))   
        a +=1
        print(a)
        for j in range(len_of_positions):
            link = "https://www.rocketpunch.com" + positions[i].select('div:nth-child(1) > a.nowrap')[j].get('href') 
            response = requests.get(link)
            dom_for_skills = BeautifulSoup(response.text,"html.parser")      
            datas.append({
                "date" : date.today(),
                "company" : root[i].select('div.company-name > a > h4 > strong')[0].text,
                "position" : positions[i].select('div:nth-child(1) > a.nowrap')[j].text,
                "salary" : positions[i].select('div.job-detail > div:nth-child(1) > span')[j].text,
                "deadline" : positions[i].select('div.job-detail > div.job-dates > span:nth-child(1)')[j].text.strip(' \n'),
                "description" : root[i].select('div.description')[0].text.strip('\xa0'),
                "service" : root[i].select('div.nowrap.meta')[0].text.strip(),  
                "link" : link,
                "skills" : str([a.text for a in dom_for_skills.select('div .job-specialties > a ')]).strip("[]")
            })
result = pd.DataFrame(datas)
url = "https://www.rocketpunch.com/api/jobs/template?&page=1&q=&specialty=Python&tag=%EB%8D%B0%EC%9D%B4%ED%84%B0"
response = requests.get(url)
data = response.json()['data']['template']
dom = BeautifulSoup(data, "html.parser")
total_pages = dom.select("div.ui.pagination.menu > div.tablet.computer.large.screen.widescreen.only > a")[-1].text
dom.select('#company-list > div.company.item > div.content > div.company-jobs-detail')[8].select('div.job-detail > div.job-dates > span:nth-child(1)')[0].text.strip(' \n').replace("/",".")[:5]
[a.text for a in dom_for_skills.select('#wrap > div.eight.wide.job-content.column > section:nth-child(3) > div > a')] 
[a.text for a in dom_for_skills.select('div .job-specialties')]
response = requests.get("https://www.rocketpunch.com/jobs/57705/Back-end-Developer-%EB%B0%B1%EC%97%94%EB%93%9C-%EC%84%9C%EB%B2%84-%EA%B0%9C%EB%B0%9C%EC%9E%90")
dom = BeautifulSoup(response.text,"html.parser")
response = requests.get("https://www.rocketpunch.com/jobs/70804/%EC%B2%AD%EB%85%84SW%EC%95%84%EC%B9%B4%EB%8D%B0%EB%AF%B8SSAFY-%EC%B7%A8%EC%97%85%EC%83%81%EB%8B%B4%EC%82%AC-%EA%B3%84%EC%95%BD%EC%A7%81-%EB%8C%80%EC%A0%84-%EA%B4%91%EC%A3%BC")
dom_for_skills = BeautifulSoup(response.text,"html.parser")      
", ".join([a.text for a in dom_for_skills.select('div .job-specialties > a ')])
# <br\>로 나누어져 있기 때문에, 처음부터 끝까지 1칸씩 간격을 두고 찾아내면 지역이 몇개건 상관없이 가져올수 있음
", ".join(dom_for_skills.select('#wrap > div.four.wide.job-infoset.column > div > div:nth-child(3) > div > div:nth-child(2) > div.content')[0].contents[::2]).strip()
"\n       광주광역시                 \n".strip()
from scrapy.http import TextResponse
url = "https://www.rocketpunch.com/api/jobs/template?page=1&q=&specialty=Python&tag=데이터"
req = requests.get(url)
response = TextResponse(req.url, body=req.text, encoding="utf-8")
response
jsonresponse = json.loads(response.body)
soup = BeautifulSoup(response.body, "html.parser") 
soup
url = "https://www.rocketpunch.com/api/jobs/template?page=1&q=&specialty=Python&tag=%EB%8D%B0%EC%9D%B4%ED%84%B0%3E"
response = requests.get(url)
data = response.json()['data']['template']
dom = BeautifulSoup(data, "html.parser")

dom
type(dom)
type(response)
response.xpath('#company-list > div.company.item.active > div.content.active > div.company-name > a:nth-child(1) > h4')
response.xpath('//*')
data = pd.read_csv('test.csv')
req = requests.get("http://www.jobkorea.co.kr/Search/?stext=%EB%8D%B0%EC%9D%B4%ED%84%B0%20%EB%B6%84%EC%84%9D&careerType=1&tabType=recruit&Page_No=1")
response = TextResponse(req.url, body=req.text, encoding='utf8')
response.xpath('//*[@id="dev_contents"]/div')<jupyter_output><empty_output><jupyter_text>잡코리아 크롤링- 무조건 xpath 사용해서!! 조건 : 신입<jupyter_code>import scrapy
from scrapy.http import TextResponse
# deadline 계산기 : 노 필요
import time
from datetime import date,timedelta
today = date.today()
deadline_date = today + timedelta(5)
"{}/{}마감".format(deadline_date.month,deadline_date.day)
http://www.jobkorea.co.kr/Search/?stext=데이터%분석&careerType=1
# 전체 리스트 페이지 가져오기
req = requests.get("http://www.jobkorea.co.kr/Search/?stext=%EB%8D%B0%EC%9D%B4%ED%84%B0&careerType=1&tabType=recruit&Page_No=3")
response = TextResponse(req.url, body=req.text, encoding="utf-8")
int(response.xpath('//*[@id="content"]/div/div/div[1]/div/div[2]/div[2]/div/div[3]/ul/li[2]/span/text()')[0].extract())
# 각 공고별 링크 불러오기
links = response.xpath('//*[@id="content"]/div/div/div[1]/div/div[2]/div[2]/div/div[1]/ul/li/div/div[2]/a/@href').extract()
links
# 링크에서 공고하나 선택
url = links[]
req = requests.get("http://www.jobkorea.co.kr//List_GI/GIB_Read.asp?GI_No=1739921&siteCode=WN")
response = TextResponse(req.url,body=req.text,encoding='utf-8')
links = ["http://www.jobkorea.co.kr" + link for link in links if "gamejob.co.kr" not in link if "&siteCode=WN" not in link]   
links
for link in links:
    req = requests.get(link)
    response = TextResponse(req.url,body=req.text,encoding='utf-8')
    try:
        print(response.xpath('//*[@id="container"]/section/div/article/div[1]/h3/span/text()')[0].extract().strip(),end="\n")
    except:
        print('error',end='\n')
req = requests.get(links[0])
response = TextResponse(req.url,body=req.text,encoding='utf-8')
# 1. company name
company_name = response.xpath('//*[@id="container"]/section/div/article/div[1]/h3/span/text()')[0].extract().strip()
company_name
for link in links:
    req = requests.get(link)
    response = TextResponse(req.url,body=req.text,encoding='utf-8')
    try:
        dead_line = response.xpath('//*[@id="tab02"]/div/article[1]/div/dl[2]/dd[2]/span/text()')[0].extract()[5:]
        print(dead_line)
    except:
        print("상시 채용")
import datetime
datetime
# 2. deadline
try:
    dead_line = response.xpath('//*[@id="tab02"]/div/article[1]/div/dl[2]/dd[2]/span/text()')[0].extract()[5:]
    print(dead_line)
except:
    print("상시 채용")
# 3. business
url = "http://www.jobkorea.co.kr" + response.xpath('//*/article[contains(@class, "artReadCoInfo") and contains(@class, "divReadBx")]/div/div/p/a/@href')[0].extract()
req = requests.get(url)
response_detail_page = TextResponse(req.url,body=req.text,encoding='utf-8')
response_detail_page.xpath('//*[@id="company-body"]/div[1]/div[1]/div/div/div[9]/div[2]/div/div/text()')[0].extract()
response.xpath('//*[@id="container"]/section/div/article/div[2]/div[3]/dl/dd/text()')[0].extract().strip()
# 4. link
print(links[0])
# 5. position
response.xpath('//*[@id="container"]/section/div/article/div[1]/h3/text()')[1].extract().strip()
# 6. job_condition
try:
    job_condition = response_detail_page.xpath('//*[@id="company-body"]/div[1]/div[1]/div/div/div[8]/div[2]/div/div/div/div/text()')[0].extract()
    print(job_condition)
except:
    print(0)
response.xpath('//*[@id="container"]/section/div/article/div[2]/div[2]/dl/dd[2]/text()')
import pymongo

client = pymongo.MongoClient('mongodb://15.164.136.109:27017')
db = client.job_position_crawler
collection = db.job_position
collection
response.xpath('//*[@id="dev_contents"]/div[2]/div[1]')
# 7. keyword
keyword = ", ".join(response.xpath('//*[@id="artKeywordSearch"]/ul/li/button/text()').extract())
keyword
# 8. location
",".join(response.xpath('//*[@id="container"]/section/div/article/div[2]/div/dl/dd/a/text()').extract())
for i in range(0,20):
    time.sleep(2)
    url = links[i]
    req = requests.get(url)
    response = TextResponse(req.url,body=req.text,encoding='utf-8')
    result = " ".join(response.xpath('//*[@id="container"]/section/div/article/div[2]/div[2]/dl/dd/span[@class="tahoma"]/text()').extract()).strip().split(" ")[0]
    if len(result) <= 2:
        result = "회사 내규에 따름"
        print(result, end="========")
    else:
        print(result, end="========")
result = " ".join(response.xpath('//*[@id="container"]/section/div/article/div[2]/div[2]/dl/dd/span[@class="tahoma"]/text()').extract()).strip().split(" ")[0]
result
if len(result) =< 2:
    result = "회사 내규에 따름"
else:
    print(result)
" ".join(response.xpath('//*[@id="container"]/section/div/article/div[2]/div[2]/dl/dd/*/text()').extract()).strip()
result = " ".join(response.xpath('//*[@id="container"]/section/div/article/div[2]/div[2]/dl/dd[2]/*/text()').extract())
if not result:
    result = "회사 내규에 따름"
result
  try:
            item["salary_condition"] = response_detail_page.xpath('//*[@id="company-body"]/div[1]/div[1]/div/div/div[8]/div[2]/div/div/div/div/text()')[0].extract()
        except:
            item["salary_condition"] = "회사 내규에 따름 - 연봉 협의"
# BeautifulSoup dom
# page_no를 돌려라
req = requests.get("http://www.jobkorea.co.kr/Search/?stext=%EB%8D%B0%EC%9D%B4%ED%84%B0&careerType=1&tabType=recruit&Page_No=1")
dom = BeautifulSoup(req.text, "html.parser")
# company 
dom.select('.name.dev_view')[0].get("title")
# description
response = requests.get(link)
dom_for_pages = BeautifulSoup(response.text, "html.parser")
dom_for_pages.select_one('.hd_4 ~ dl.tbList').text.strip('\n ')
# service
dom_for_pages.select_one("#container > section > div > article > div.tbRow.clear > div.tbCol.tbCoInfo > dl.tbList").text.split('\n')
# position
dom.select('.post-list-info > a')[0].get('title')
# link
link = "http://www.jobkorea.co.kr" + dom.select('.post-list-info > a')[15].get('href')
# deadline
dom.select('.post-list-info > p[class="option"]')[0].select_one('span[class=date]').text.strip('(~)월화수목금토일')+"마감"
# task & skills
dom.select('p.etc')[0].text
# jon_condition
req2 = requests.get("http://www.jobkorea.co.kr/Salary/SearchOfSalary/671394")
dom = BeautifulSoup(req2.text, "html.parser")
salary = dom.select_one('div.salary-table-item.salary-table-newcomer > div.salary > div.value').text
salary<jupyter_output><empty_output><jupyter_text>사람인- keyword : 데이터 사이언스- 모바일 페이지 탐색중<jupyter_code>import re
import string
url= "http://www.saramin.co.kr/zf_user/search/recruit?searchType=search&company_cd=0,1,2,3,4,5,6,7,9&searchword=%EB%8D%B0%EC%9D%B4%ED%84%B0%20%EC%82%AC%EC%9D%B4%EC%96%B8%EC%8A%A4&panel_type=&search_optional_item=y&search_done=y&panel_count=y&recruitPage=2&recruitSort=relation&recruitPageCount=40&inner_com_type="
req = requests.get(url)
response = TextResponse(req.url, body=req.text, encoding="utf-8")
url = "http://m.saramin.co.kr/search?searchType=search&searchword=데이터"
req = requests.get(url)
response = TextResponse(req.url, body=req.text, encoding="utf-8")
total_page = int(response.xpath('//*[@id="recruit_total_count"]/text()')[0].extract().replace(',','')) // 20
total_page
total_links =[]
for i in range(1,int(total_page)+1):
    url = url + "&page={}".format(i)
    req = requests.get(url)
    response = TextResponse(req.url, body=req.text, encoding="utf-8")
    links = response.xpath('//*[@id="recruit_result"]/ul/li/div/a/@href').extract()
    links = ["http://m.saramin.co.kr" + link + "&t_ref=search&t_ref_content=generic" for link in links]
    total_links += links
req = requests.get(total_links[2000])
response = TextResponse(req.url,body=req.text,encoding='utf8')
response
# company_name 
company_name = response.xpath('//*[@id="card_set"]/div[1]/div/div[1]/div/span/a/text()').extract()
company_name
response.xpath('//*[@id="card_set"]/div[1]/div/div[1]/div/span/span/text()')[0].extract()
# business 
response.xpath('//*[@id="card_set"]/div[1]/div/section[4]/div/div[2]/dl[2]/dd/div/text()').extract()
# position
response.xpath('//*[@id="card_set"]/div[1]/div/div[1]/div/h2/text()')[0].extract().strip()
# link
link = total_links[11]
print(link)
# salary_condition
req = requests.get(total_links[i])
response = TextResponse(req.url,body=req.text,encoding='utf8')
try:
    result = response.xpath('//*[@class="card_cont wrap_recruit_view"]/section[@class="section_view section_basic_view"]/div[@class="wrap_info_job"]/dl[2]/dd/text()')[0].extract().strip()
    if result:
        print(result)
    else:
        print("면접 후 결정 (조건 상이)")
except:
    print("연봉 조건 정보 없음")
print(total_links[0])
# location
location = response.xpath('//*[@class="card_cont wrap_recruit_view"]/section[@class="section_view section_basic_view"]/div[@class="wrap_info_job"]/dl[3]/dd/div/text()')[0].extract().strip()
location
# deadline
deadline = response.xpath('//*[@class="card_cont wrap_recruit_view"]/section[@class="section_view section_basic_view"]/div[@class="wrap_info_job"]/dl[1]/dd/span[1]/text()')[0].extract().strip()
deadline
# keyword
keyword = str(response.xpath('//*[@class="list_relation_tag"]/li/a/text()').extract()).strip("[]''")
keyword
link = total_links[0]
print(link)
for link in total_links[:20]:
    req = requests.get(link)
    response = TextResponse(req.url, body=req.text, encoding="utf-8")
    result = response.xpath('//*[@id="card_set"]/div[1]/div/div[1]/div/span/span/text()').extract()
    if result:
        print(response.xpath('//*[@id="card_set"]/div[1]/div/div[1]/div/span/span/text()').extract(),end="\n")
    else :
        print(response.xpath('//*[@id="card_set"]/div[1]/div/div[1]/div/span/a/text()').extract(),end="\n")
total_links[3]
req = requests.get(link)
response = TextResponse(req.url, body=req.text, encoding="utf-8")
response.xpath('//*[@id="card_set"]/div[1]/div/div[1]/div/span/a/text()').extract()
for link in total_links[:20]:
    req = requests.get(link)
    response = TextResponse(req.url, body=req.text, encoding="utf-8")
    result = response.xpath('//*[@id="card_set"]/div[1]/div/div[1]/div/span/span/text()').extract()
    if result:
        print(response.xpath('//*[@id="card_set"]/div[1]/div/div[1]/div/span/span/text()').extract(),end="\n")
    else :
        print(response.xpath('//*[@id="card_set"]/div[1]/div/div[1]/div/span/a/text()').extract(),end="\n")
# business
print(total_links[3])
req = requests.get(total_links[0])
response = TextResponse(req.url, body=req.text, encoding="utf-8")
for link in total_links:
    req = requests.get(link)
    response = TextResponse(req.url, body=req.text, encoding="utf-8")
    try:
        print(response.xpath('//*[@id="card_set"]/div[1]/div/section[4]/div/div[2]/dl[2]/dd/div/text()')[0].extract(),end="\n")
    except:
        print("error",end="\n")
print(total_links[2])
print(total_links[2])
print("http://m.saramin.co.kr/job-search/view?rec_idx=37863553&cn=search&cns=index&t_category=search&t_content=generic&searchword=%EB%8D%B0%EC%9D%B4%ED%84%B0+%EC%82%AC%EC%9D%B4%EC%96%B8%EC%8A%A4&searchType=search&paid_fl=n&t_ref=search&t_ref_content=generic")
print(total_links[22])
for l in total_links:
    req = requests.get(l)
    response = TextResponse(req.url, body=req.text, encoding="utf-8")
    try:
        print(response.xpath('//*[@class="card_cont wrap_recruit_view"]/section[@class="section_view section_basic_view"]/div[@class="wrap_info_job"]/dl[1]/dd/span[1]/text()')[0].extract().strip(),end='\n')
    except:
        print("error",end='\n')
response.xpath('//*[@id="card_set"]/div[1]/div/section[4]/div/div[2]/dl[2]/dd/div/text()')[0].extract().strip()
response.xpath('//*[@id="card_set"]/div[1]/div/div[1]/div/h2/text()')[0].extract().strip()
response.xpath('//*[@id="card_set"]/div[1]/div/div[1]/div/span/span/text()').extract()
response.xpath('//*[@class="footer_link"]/li/a/@href')[2].extract()
for l in total_links:
    req = requests.get(l)
    response = TextResponse(req.url, body=req.text, encoding="utf-8")
    
    result = response.xpath('//*[@id="card_set"]/div[1]/div/div[1]/div/span/span/text()').extract()
    if result:
        print(result[0],end="\n")
    else :
        print(response.xpath('//*[@id="card_set"]/div[1]/div/div[1]/div/span/a/text()').extract()[0],end="\n")<jupyter_output>(주)이음인터렉티브
(주)디세이코리아
(주) 컴투스
(주)시몬스
유사나헬스사이언스코리아(유)
패스트캠퍼스
MKS Korea
브랭섬홀아시아(BHA)
(주)파블로항공
주식회사 에비드넷
주식회사 대교씨엔에스
(주)라이언앤코
(주)라이언앤코
(주)라이언앤코
(주)SPH
(주)러닝스푼즈
지이헬스케어코리아(주)
(주)레코드라인
(주)글로벌모터서비스
렌도코리아 
건양대학교병원
스마트마인드(주)
(주)지티씨솔루션
(주)투미유
주식회사 강남엄마
주식회사 에이티지
이에이치알앤씨(주)
입소스 주식회사(IPSOS)
존슨앤드존슨
(주)존슨앤드존슨
한국피앤지판매유한회사
트랜스퍼펙트코리아(유)
(재)한국세계자연기금
글로벌라이프사이언스솔루션즈코리아(유)
엠씨지컨설팅(주)
렌도코리아 
밀리언웨어
(주)동아사이언스 
(주)동아사이언스 
씨비에스바이오사이언스(주)
퓨처메인(주)
(주)엠서클
주식회사 윙블링
(주)제이앤케이사이언스
(주)엠폴시스템
(주)디케이비엠시
(주)에필
(주)제이엔피글로벌
더넥스트스쿨(주)
한국암웨이(주)
한국암웨이(주)
한영회계법인
한영회계법인
(주)파수닷컴
인피니언테크놀로지스파워세미텍(주)
갈더마코리아(주)
한국테라데이타(유)
(주)씨앤테크
(주)레인보우브레인	
피피씨코리아(주)
주식회사 강남엄마
자이랜드(주)
(주)에이조스바이오
라인파이낸셜플러스(주)
피티크래프트
(주)글로벌트러스트네트웍스코리아
{'fraction_non_alphanumeric': 0.21792269305590797, 'fraction_numerical': 0.024824494629112747, 'mean_word_length': 4.693956176258126, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 179, 'https://': 16, 'lorem ipsum': 0, 'www.': 26, 'xml': 0}, 'pii_count': 1, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '101735', 'license': 'MIT', 'n_tokens_mistral': 10032, 'n_tokens_neox': 10552, 'n_words': 1123, 'path': 'rocketpunch_crawler/crawling_test.ipynb', 'repo_name': 'HowardHowonYu/Crawling_project', 'size': '23646'}