<jupyter_start><jupyter_text>testing box conversion<jupyter_code>import requests
from requests.auth import HTTPBasicAuth
import json
import datetime
from impresso_commons.path.path_fs import IssueDir, detect_canonical_issues, canonical_path, KNOWN_JOURNALS
from text_importer.helpers import get_page_schema
from impresso_commons.images.olive_boxes import get_iiif_url
from IPython.display import Image
issues = detect_canonical_issues("/scratch/matteo/images_test/", KNOWN_JOURNALS)
len(issues)
import os
import json
from random import randint, shuffle

Page = get_page_schema()

def select_random_index(objects):
    shuffle(objects)
    total_objects = len(objects)
    index = randint(0, len(objects) - 1)
    return index

def read_issue_pages(issue):
    pages = []
    
    page_files = [
        file
        for file in os.listdir(issue.path)
        if not 'issue.json' in file
    ]
    
    for page_file in page_files:
        
        page_path = os.path.join(issue.path, page_file)
        
        with open(page_path, 'r') as f:
            data = json.loads(f.read())
            pages.append((page_file, Page(**data)))
    
    return pages

def select_random_token(page_obj):
    total_regions = len(page_obj.r)
    region = page_obj.r[randint(0, total_regions - 1)]
    total_paragraphs = len(region.p)
    paragraph = region.p[randint(0, total_paragraphs - 1)]
    total_lines = len(paragraph.l)
    line = paragraph.l[randint(0, total_lines - 1)]
    total_tokens = len(line.t)
    token = line.t[randint(0, total_tokens - 1)]
    return token

def get_link(page_name, coordinates):
    link = get_iiif_url(page_name, " ".join([str(i) for i in coordinates]))
    return link

def get_image(page_name, coordinates, user="epfl-team", pwd="_<PASSWORD>"):
    link = get_iiif_url(page_name, " ".join([str(i) for i in coordinates]))
    return requests.get(link, auth=HTTPBasicAuth(user, pwd)).content
    
for n in range(0, 40):
    issue_idx = select_random_index(issues)
    issue = issues[issue_idx]
    print(issue.path)
    pages = read_issue_pages(issue)
    
    for j in range(0, 5):
        page_idx = select_random_index(pages)
        page_name, page = pages[page_idx]
        print(page_name)

        for i in range(0, 2):
            try:
                token = select_random_token(page)
                print(token.tx, list(token.c))
                print(get_link(page_name.replace('.json', ''), token.c))
            except Exception as e:
                print(e)
                pass
    
#Image(get_image("GDL-1900-01-02-a-p0001", token.c))<jupyter_output><empty_output><jupyter_text>data acquisition<jupyter_code>import json
from impresso_commons.path import *
from text_importer.helpers import get_page_schema
Page = get_page_schema()
issues = detect_canonical_issues('./text_importer/data/out/',"GDL")
pages = []
for issue in issues[:1]:
    page_files = [
        f
        for f in os.listdir(issue.path)
        if "issue" not in f
    ]
    for file in page_files:
        file_path = os.path.join(issue.path, file)
        with open(file_path, 'r') as inp_file:
            json_data = json.load(inp_file)
            pages.append(Page(**json_data))
for region in pages[0].r:
    print("[Part of item {}]".format(region.pOf))
    for paragraph in region.p:
        for line in paragraph.l:
            tokens = [str(token.tx) for token in line.t]
            print(" ".join(tokens))
    print("\n")<jupyter_output>[Part of item GDL-1900-01-02-a-i0001]
RÉDACTION
B<NAME> i
BUREAU D'ABONNEMENTS
Ruelle Si-François , 20-
Oo s'abouno dans tous les bureaux de posta
Les abonnements partent
du i" ou du 15 de chaque m » . *


[Part of item GDL-1900-01-02-a-i0002]
Gazelle de Lausanne


[Part of item GDL-1900-01-02-a-i0002]
ET JOURNAL SUISSE
FONDÉE EN 1798


[Part of item GDL-1900-01-02-a-i0003]
PRIX D'ABONNEMENl
Ul U ( BtÛ I Mit
Strisse Fr . 20 10 50 5 50
Union postale » 36 18 50 9 50
Prix du numéro : 10 centimes .


[Part of item GDL-1900-01-02-a-i0004]
Lausanne , 2 janvier 1900 .
Regards en arrière .


[Part of item GDL-1900-01-02-a-i0004]
Supposez un dessinateur installé au coin
de son feu tandis que la dernière bûche de
l'année s'effondrait bleuâtre entre les che-
nets . Supposez-le cherchant à personnifier
le peuple suisse , comme d'au 1 res ont
évoqué en « oncle Sam » les Yankees , ou ,
en <NAME> , la nation britannique . La
silhouette découlant de la pointe de son
crayon aurait été environ la [...]
{'fraction_non_alphanumeric': 0.10448430493273543, 'fraction_numerical': 0.03071748878923767, 'mean_word_length': 3.111520737327189, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '99288', 'license': 'MIT', 'n_tokens_mistral': 1664, 'n_tokens_neox': 1554, 'n_words': 412, 'path': 'notebooks/test_text_boxes.ipynb', 'repo_name': 'aflueckiger/impresso-text-acquisition', 'size': '4471'}