<commit_before><commit_msg>first import of metadata writer<commit_after>import nbformat
from nbformat import NotebookNode

test = 'what_is_data_1.ipynb'
out = 'test.ipynb'


data = nbformat.read(test, as_version=nbformat.NO_CONVERT)
new_metadata = data.metadata.copy()

cells = data.cells

document = NotebookNode(dict(author = '<NAME>', licence = 'CC-BY', title="What is Data???"))

new_metadata['document'] = document
data.metadata = NotebookNode(new_metadata)
nbformat.validate(data)

header = cells[0]['source']


title = '# ' + document.title
author = document.author

header_lines = header.split('\n')


if header_lines[0].startswith('#'):
    del header_lines[0]

header_lines = [title, author] + header_lines
header_string = '\n'.join(header_lines)

data.cells[0]['source'] = header_string

with open(out,  encoding='utf-8', mode='w') as fp:
    nbformat.write(data, fp)





{'fraction_non_alphanumeric': 0.14835787089467722, 'fraction_numerical': 0.006795016987542469, 'mean_word_length': 5.696969696969697, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3492919', 'n_tokens_mistral': 312, 'n_tokens_neox': 301, 'n_words': 62}