<jupyter_start><jupyter_text>Get Old OCR<jupyter_code>import os
from pprint import pprint

import numpy as np
import pandas as pd<jupyter_output><empty_output><jupyter_text>Figures from database<jupyter_code>%sql postgresql:///pfocr20200224
%%sql old_ocr << SELECT pmcid, figure_number, result->'textAnnotations'->0->'locale' AS locale, result->'textAnnotations'->0->'description' AS description
FROM ocr_processors__figures
INNER JOIN ocr_processors ON ocr_processors__figures.ocr_processor_id = ocr_processors.id
INNER JOIN figures ON ocr_processors__figures.figure_id=figures.id
INNER JOIN papers ON figures.paper_id = papers.id
LIMIT 5;
old_ocr_df = old_ocr.DataFrame()

old_ocr_df.head()
%%sql biothings_export << SELECT pmcid, figure_number,
word, symbol, xref AS entrez
FROM figures
INNER JOIN papers ON figures.paper_id = papers.id
INNER JOIN match_attempts ON figures.id = match_attempts.figure_id
LEFT JOIN symbols ON match_attempts.symbol_id = symbols.id
LEFT JOIN transformed_words ON match_attempts.transformed_word_id = transformed_words.id
LEFT JOIN lexicon ON match_attempts.symbol_id = lexicon.symbol_id
LEFT JOIN xrefs ON lexicon.xref_id = xrefs.id
WHERE xref IS NOT NULL;
bte_figures_df = biothings_export.DataFrame()
bte_figures_df["filename"] = bte_figures_df["figure_number"] + ".jpg"
bte_figures_df["figid"] = (
    bte_figures_df["pmcid"] + "__" + bte_figures_df["filename"]
)
bte_figures_df["url"] = (
    "https://www.ncbi.nlm.nih.gov/pmc/articles/"
    + bte_figures_df["pmcid"]
    + "/bin/"
    + bte_figures_df["filename"]
)

bte_figures = set(bte_figures_df["figid"])

print(len(bte_figures))
print(list(bte_figures)[0:5])
bte_figures_df.head()
bte_figures_df.to_csv(
    f"~/code/pathway-ocr/20191102_100k/pfocr_biothings.tsv",
    sep="\t",
    index=False,
)
os.system("jupyter nbconvert --to html biothings_export.ipynb")
os.system("mv biothings_export.html ~/code/pathway-ocr/20191102_100k/")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11491317671092952, 'fraction_numerical': 0.01787538304392237, 'mean_word_length': 6.448669201520913, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 15, 'https://': 1, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '276121', 'license': 'Apache-2.0', 'n_tokens_mistral': 786, 'n_tokens_neox': 722, 'n_words': 137, 'path': 'notebooks/get_raw_2019_ocr_data.ipynb', 'repo_name': 'wikipathways/pathway-figure-ocr', 'size': '1958'}