<jupyter_start><jupyter_text>KCP Dataset PDF Cover Page Snapshot<jupyter_code>%load_ext autotime
from wb_nlp import dir_manager
!/home/wb536061/anaconda3/envs/wb_nlp/bin/pip install pdf2image > /dev/null
import requests
import json
import os
import time
import glob
from pathlib import Path
import pandas as pd
import re
from joblib import Parallel, delayed
import pdf2image
import PIL

class DocumentCover:
    def __init__(self, doc_id, cover_dir, pdf_path=None, pdf_url=None, fixed_width=200, fixed_height=None):
        self.doc_id = doc_id
        self.pdf_path = pdf_path
        self.pdf_url = pdf_url
        self.cover_dir = cover_dir
        self.fixed_width = fixed_width
        self.fixed_height = fixed_height

        # self.fname = os.path.join(self.cover_dir, f'{self.doc_id}.png')
        self.fname = self.cover_dir / f'{self.doc_id}.png'

        self.orig_cover = None
        self.cover = None
        self.resized = None

    def get_content_from_url(self):
        self.resized = False
        res = requests.get(self.pdf_url)

        try:
            pages = pdf2image.convert_from_bytes(res.content, size=(self.fixed_width, self.fixed_height), single_file=True)
            self.resized = True
        except:
            pages = pdf2image.convert_from_bytes(res.content)
            
        self.orig_cover = pages[0]

    def get_content_from_file(self):
        self.resized = False
        with open(self.pdf_path, "rb") as open_file:
            content = open_file.read()
            try:
                pages = pdf2image.convert_from_bytes(content, size=(self.fixed_width, self.fixed_height), single_file=True)
                self.resized = True
            except:
                pages = pdf2image.convert_from_bytes(content)

        self.orig_cover = pages[0]

    def standardize_size(self):
        # Not useful if `size` is set in pdf2image.convert_from_bytes
        c = self.orig_cover

        if c is None:
            if self.pdf_path:
                self.get_content_from_file()
            elif self.pdf_url:
                self.get_content_from_url()
            c = self.orig_cover
            assert(c)

        if not self.resized:
            width = self.fixed_width
            w0, h0 = c.size
            c = c.resize((width, int(h0 * width / w0)), resample=PIL.Image.BICUBIC)

        self.cover = c

    def save(self):
        if not os.path.isfile(self.fname):        
            cover = self.cover

            if cover is None:
                self.standardize_size()
                cover = self.cover

            cover.save(self.fname)

        return self.doc_id

    def cleanup(self):
        del(self.orig_cover)
        del(self.cover)

        self.orig_cover = None
        self.cover = None
corpus_path = Path(dir_manager.get_data_dir("corpus").replace("/home/", "/data/"))
CORPUS_DIRS = list(corpus_path.glob("*/PDF_ORIG"))
def prepare_meta(corpus_dir):
    data = []
    corpus_id = corpus_dir.parent.name.lower()

    for pdf in corpus_dir.glob("*.pdf"):
        metadata = {}

        if pdf.stem.startswith(f"{corpus_id}_"):
            doc_id = pdf.stem
        else:
            doc_id = f"{corpus_id}_{pdf.stem}"

        metadata["id"] = doc_id
        metadata["pdf_path"] = pdf.resolve()
        metadata["cover_dir"] = (corpus_dir.parent / "COVER").resolve()
        metadata["corpus_id"] = corpus_id

        data.append(metadata)

    return data
def parallel_process(row):
    ret = dict(status='ok', doc_id=row['id'], corpus_id=row["corpus_id"], pdf_path=str(row["pdf_path"]))
    try:
        dc = DocumentCover(doc_id=row['id'], cover_dir=row["cover_dir"], pdf_path=row["pdf_path"])
        r = dc.save()
        dc.cleanup()
    except Exception as e:
        ret['status'] = e.__str__()

    return ret
dataset = []
for corpus_dir in CORPUS_DIRS:
#     if corpus_dir.parent.name == "IADB":
#         continue
    COVER_DIR = corpus_dir.parent / "COVER"
    if not COVER_DIR.exists():
        COVER_DIR.mkdir(parents=True)

    dataset.extend(prepare_meta(corpus_dir))
len(dataset)
# Test
row = dataset[75000]
# row = dataset[0]
print(row)
dc = DocumentCover(doc_id=row['id'], cover_dir=row["cover_dir"], pdf_path=row['pdf_path'])
dc.standardize_size()
dc.cover
NUM_JOBS = 70

pres = Parallel(n_jobs=NUM_JOBS)(delayed(parallel_process)(row) for row in dataset)<jupyter_output>time: 22min 57s<jupyter_text>Review rate of unprocessed files, if any<jupyter_code>pres_df = pd.DataFrame(pres)
pres_df.head()
100 * (pres_df[pres_df["status"] != "ok"]["corpus_id"].value_counts() / pres_df["corpus_id"].value_counts()).dropna().sort_values(ascending=False)
pres_df[pres_df["status"] != "ok"]["corpus_id"].value_counts()
pres_df["corpus_id"].value_counts()
# !ls -l /R/NLP/CORPUS/WB/cover | wc -l
!df -h
!df -h
# You can pass it as a build arg. Your Dockerfile can be static:

# FROM ubuntu:xenial-20170214
# ARG UNAME=testuser
# ARG UID=1000
# ARG GID=1000
# RUN groupadd -g $GID -o $UNAME
# RUN useradd -m -u $UID -g $GID -o -s /bin/bash $UNAME
# USER $UNAME
# CMD /bin/bash
# Then you'd pass the options on your build command:

# docker build --build-arg UID=$(id -u) --build-arg GID=$(id -g) \
#   -f bb.dockerfile -t testimg .
from pathlib import Path
# corpus_dir = Path('/data/wb536061/wb_nlp/data/corpus')
corpus_dir = Path('/workspace/data/corpus')

base_dirs_ext = {
    "TXT_ORIG": ".txt",
    "PDF_ORIG": ".pdf",
    "NON_EN_TXT_ORIG": ".txt",
    "EN_TXT_ORIG": ".txt",
    "COVER": ".png",  # Naming alread fixed.  
}

for base_dir, ext in base_dirs_ext.items():
    print(base_dir)
    for path in corpus_dir.glob(f"*/{base_dir}/*{ext}"):
        if path.is_file():
            corpus_id = path.parent.parent.name.lower()
            if path.name.startswith(f"{corpus_id}_"):
                continue
            new_name = path.parent / f"{corpus_id}_{path.name}"
            # print(new_name)
            path.rename(new_name)
from pathlib import Path
# corpus_dir = Path('/data/wb536061/wb_nlp/data/corpus')
corpus_dir = Path('/workspace/data/corpus')

base_dirs_ext = {
    "TXT_ORIG": ".txt",
    "PDF_ORIG": ".pdf",
    "NON_EN_TXT_ORIG": ".txt",
    # "EN_TXT_ORIG": ".txt",
    "COVER": ".png",  # Naming alread fixed.  
}

for path in corpus_dir.glob(f"WB/COVER/*.png"):
    if path.is_file():
        corpus_id = path.parent.parent.name.lower()
        if path.name.startswith(f"{corpus_id}_{corpus_id}_"):
            new_name = path.parent / "_".join(path.name.split("_")[1:])
#             print(new_name)
            path.rename(new_name)
#     break
path<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13204118715929739, 'fraction_numerical': 0.010145366444579043, 'mean_word_length': 2.6920067076579093, 'pattern_counts': {'":': 11, '<': 8, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '555093', 'license': 'MIT', 'n_tokens_mistral': 2414, 'n_tokens_neox': 2318, 'n_words': 454, 'path': 'notebooks/pdf-cover/pdf_cover_scraper.ipynb', 'repo_name': 'worldbank/wb-nlp-apps', 'size': '6604'}