<jupyter_start><jupyter_text>Proof of Concept- convert text document to black/white image input.<jupyter_code>%config IPCompleter.greedy=True
import warnings

warnings.filterwarnings('ignore')
import numpy as np
%%html

<h5>1. setup</h5>
document = '\n' \
     + '     abc: 10     abc: 10     abc: 10' + '\n' \
     + '     abc: 10     abc: 10     abc: 10' + '\n' \
     + '     abc: 10     abc: 10     abc: 10' + '\n' \
     + '     abc: 10     abc: 10     abc: 10' + '\n' \
     + '\n' \
     + '-----------------------------------------' + '\n' \
     + '\n' \
     + '     abc: 10     abc: 10     abc: 10' + '\n' \
     + '     abc: 10     abc: 10     abc: 10' + '\n' \
     + '     abc: 10     abc: 10     abc: 10' + '\n' \
     + '     abc: 10     abc: 10     abc: 10' + '\n' \

print('example document:')
print('-----------------------------------------')
print(document)
print('-----------------------------------------')
%%html

<h5>2. convert document to matrix</h5>
lines_in_document = document.split('\n')
## shape
length = len(lines_in_document)
width = np.max(
    [
        len(line)
        for line
        in lines_in_document
    ]
)

document_as_matrix = np.zeros((length, width))
for r, line in enumerate(lines_in_document):
    for c, char in enumerate(line.replace('\s', ' ')):
        if char != ' ':
            document_as_matrix[r, c] = 1
%%html

<h5>3. image / document</h5>
import matplotlib.pyplot as plt
print('example document:')
print('-----------------------------------------')
print(document)
print('-----------------------------------------')

plt.imshow(document_as_matrix)<jupyter_output>example document:
-----------------------------------------

     abc: 10     abc: 10     abc: 10
     abc: 10     abc: 10     abc: 10
     abc: 10     abc: 10     abc: 10
     abc: 10     abc: 10     abc: 10

-----------------------------------------

     abc: 10     abc: 10     abc: 10
     abc: 10     abc: 10     abc: 10
     abc: 10     abc: 10     abc: 10
     abc: 10     abc: 10     abc: 10

-----------------------------------------
{'fraction_non_alphanumeric': 0.28743961352657005, 'fraction_numerical': 0.051207729468599035, 'mean_word_length': 1.2031914893617022, 'pattern_counts': {'":': 0, '<': 10, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '573136', 'license': 'MIT', 'n_tokens_mistral': 752, 'n_tokens_neox': 649, 'n_words': 186, 'path': 'notebooks/poc_convert_text_to_image.ipynb', 'repo_name': 'dpasse/summer_of_george', 'size': '2070'}