<jupyter_start><jupyter_text>Help download html from needCrawl.csv<jupyter_code>import pandas as pd
import sys
import os
sys.path.insert(0, os.path.abspath(r".."))
from autopager.preprocessing import generate_page_component
file = pd.read_csv('./needCrawl.csv')
file = file[['File Name','Page URL']]
file
!ls ./download
for idx, row in file.iterrows():
    file_idx = row['File Name']
    page_url = row['Page URL']
    print(f"Now downloading page {file_idx}: {page_url}")
    try:
        page_html = generate_page_component(page_url)['html']
        with open(f"./download/{str(file_idx)}.html", 'w') as f:
            f.write(page_html)
    except Exception as e:
        print(f"page idx: {file_idx} have error ({e})")
    print(f"Finish saving page {file_idx}: {page_url}")
!ls ./download<jupyter_output>100.html  57.html  64.html  71.html  78.html  85.html  92.html	99.html
51.html   58.html  65.html  72.html  79.html  86.html  93.html
52.html   59.html  66.html  73.html  80.html  87.html  94.html
53.html   60.html  67.html  74.html  81.html  88.html  95.html
54.html   61.html  68.html  75.html  82.html  89.html  96.html
55.html   62.html  69.html  76.html  83.html  90.html  97.html
56.html   63.html  70.html  77.html  84.html  91.html  98.html
{'fraction_non_alphanumeric': 0.13449367088607594, 'fraction_numerical': 0.08069620253164557, 'mean_word_length': 3.0938511326860842, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 5, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '18490', 'license': 'MIT', 'n_tokens_mistral': 543, 'n_tokens_neox': 488, 'n_words': 118, 'path': 'queue_app/fileDownloader.ipynb', 'repo_name': 'TsuNiWan/pagination-prediction', 'size': '1264'}