<jupyter_start><jupyter_text>Intel Xeon CPU 정보 수집용 프로그램 - https://en.wikipedia.org/wiki/List_of_Intel_Xeon_microprocessors 사이트 정보 이용   필요 라이브러리 호출<jupyter_code>import pandas as pd
import urllib.request as req
from bs4 import BeautifulSoup<jupyter_output><empty_output><jupyter_text>메인 페이지에서 요약 정보 추출하기<jupyter_code>url = "https://en.wikipedia.org/wiki/List_of_Intel_Xeon_microprocessors"
data = req.urlopen(url).read()
soup = BeautifulSoup(data, "html.parser")
#soup.select("h2 > span.mw-headline")
#soup.select("div > ul > li > a")
excel = pd.DataFrame()

idx = 0
base_url = dict()

data_tags = soup.select("div > ul > li")
for data_tag in data_tags:  
  try:
    url = data_tag.select_one('a')['href']
    title = data_tag.select_one('a').get('title')
    model = data_tag.select_one('a').string
  except:
    continue

  if (url[0:10]!="/wiki/List") :
    continue
  if (url.find('#')<0) :
    continue
  url = "https://en.wikipedia.org"+url
  attr1 = title.split("of Intel ")
  attr2 = attr1[1].split("-based ")
  base = attr2[0]

  url1 = url.split("#")
  burl = url1[0]
  if burl not in base_url:
    base_url[burl]=burl

  #print(url, "|", title, "|", base, "|", model)
  d=pd.DataFrame()  
  d.loc[idx, 'CPU Model']=model
  d.loc[idx, 'CPU Generation']=base
  d.loc[idx, 'URL']=url
  idx=idx+1
  excel = excel.append(d)

excel.to_excel("CPU_Model_Summary.xls")
excel<jupyter_output><empty_output><jupyter_text>상세 페이지에서 세부 정보 가져오기<jupyter_code>allexcel = pd.DataFrame()

for key in base_url:
  print(key)
  wikidata = req.urlopen(key).read()
  wikisoup = BeautifulSoup(wikidata, "html.parser")

  table = wikisoup.select("table.wikitable")
  for i in range(0,len(table)):
    df = pd.read_html(str(table))[i]

    if "Modelnumber" in df:
      df.rename(columns={'Modelnumber':'Model Number'}, inplace=True)
    if "sSpec Number" in df:
      df.rename(columns={'sSpec Number':'sSpecnumber'}, inplace=True)
    if "L2cache" in df:
      df.rename(columns={'L2cache':'L2-Cache'}, inplace=True)
    if "L2 Cache" in df:
      df.rename(columns={'L2 Cache':'L2-Cache'}, inplace=True)
    if "L3cache" in df:
      df.rename(columns={'L3cache':'L3-Cache'}, inplace=True)
    if "L3 Cache" in df:
      df.rename(columns={'L3 Cache':'L3-Cache'}, inplace=True)
    if "FSB" in df:
      df.rename(columns={'FSB':'Front Side Bus'}, inplace=True)
    if "Mult." in df:
      df.rename(columns={'Mult.':'Multiplier'}, inplace=True)
    if "Mult" in df:
      df.rename(columns={'Mult':'Multiplier'}, inplace=True)
    if "Release date" in df:
      df.rename(columns={'Release date':'Release Date'}, inplace=True)
    if "Partnumber(s)" in df:
      df.rename(columns={'Partnumber(s)':'Part Number(s)'}, inplace=True)
    if "Releaseprice (USD)" in df:
      df.rename(columns={'Releaseprice (USD)':'Release Price (USD)'}, inplace=True)
    if "Turbo Boostall-core/2.0(/max. 3.0)" in df:
      df.rename(columns={'Turbo Boostall-core/2.0(/max. 3.0)':'Turbo Boost'}, inplace=True)
    if "Turbo" in df:
      df.rename(columns={'Turbo':'Turbo Boost'}, inplace=True)
    if "Cores" in df:
      df.rename(columns={'Cores':'Cores(threads)'}, inplace=True)

    if True in df.columns.str.contains("Unnamed"):
      for n in df.columns:
        if n.find("Unnamed")>=0:          
          del df[n]
   
    allexcel = pd.concat([allexcel, df], ignore_index=True, join='outer')    

allexcel = allexcel.drop( allexcel[allexcel['Model Number']==allexcel['sSpecnumber']].index )
allexcel.sort_index(axis=1, inplace=True)
allexcel = allexcel.reindex(columns=(['Model Number'] + list([a for a in allexcel.columns if a != 'Model Number']) ))
allexcel.to_excel("CPU_Model_All.xls")
allexcel
# allexcel = allexcel.drop(allexcel[allexcel['Model Number'].str.contains("core")].index)
# allexcel = allexcel.drop(allexcel[allexcel['Model Number'].str.contains("Core")].index)
# allexcel = allexcel.drop(allexcel[allexcel['Model Number'].str.contains("standard power")].index)
# allexcel = allexcel.drop(allexcel[allexcel['Model Number'].str.contains("Edge Server and Cloud")].index)
# allexcel = allexcel.drop(allexcel[allexcel['Model Number'].str.contains("Network")].index)

# allexcel = allexcel.drop( allexcel[allexcel['Model Number']==allexcel['sSpecnumber']].index )
# allexcel.to_excel("CPU_Model_All.xls")
# allexcel

# if True in allexcel.columns.str.contains("Unnamed"):
#   print("Unnamed")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.17516428733288011, 'fraction_numerical': 0.009064128710627692, 'mean_word_length': 4.55919395465995, 'pattern_counts': {'":': 0, '<': 14, '<?xml version=': 0, '>': 20, 'https://': 3, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '574709', 'license': 'MIT', 'n_tokens_mistral': 1736, 'n_tokens_neox': 1771, 'n_words': 309, 'path': '2020/Intel_Xeon_Model.ipynb', 'repo_name': 'sukmoonlee/JupyterNotebook', 'size': '4413'}