<jupyter_start><jupyter_text>CI/CD - Automatically update documentation with templates **Tags:** naas **Author:** [<NAME>](https://www.linkedin.com/in/maximejublou/)  Input<jupyter_code>!pip install rich pydash nbconvert<jupyter_output><empty_output><jupyter_text>Import librairies<jupyter_code>import json
import glob
from rich import print
import os
import shutil<jupyter_output><empty_output><jupyter_text>Model<jupyter_code>def get_technology_and_action(filename):
    technology_name = file.split('/')[2]
    action = file.split('/')[3].replace('.ipynb', '').replace('_', ' ')
    
    if action.startswith(technology_name):
        action = action[len(technology_name):]
        
    return (technology_name.strip(), action.strip())

def filepath_to_md_path(filepath):
    return filepath.split("/")[-1].replace(".ipynb", ".md")

def create_readme_lines(notebooks):
    sorted_notebooks = list(notebooks.keys())
    sorted_notebooks.sort()
    
    lines = []
    for technology in sorted_notebooks:
        lines.append(f'  * [{technology}](templates/{technology.replace(" ", "%20")}/README.md)\n')
        for action, filepath in notebooks[technology]:
            lines.append(f'    * [{action}](templates/{technology.replace(" ", "%20")}/{filepath_to_md_path(filepath).replace(" ", "%20")})\n')
    return lines

def remove_unwanted_cells(notebooks):
    for category in notebooks:
        for nb in notebooks[category]:
            with open(nb[1], 'r') as f:
                content = json.load(f)
                content['cells'] = content['cells'][1:]
                content['cells'][0]['source'] = content['cells'][0]['source'][1:]
                with open(nb[1], 'w') as target:
                    json.dump(content, target)
                    target.close()
                    print(f'✅ Removed title and naas logo on [{nb[1]}]')

def convert_notebooks_to_md(notebooks):
    os.system('jupyter nbconvert --to markdown ../../**/*.ipynb')

def move_markdown_files():
    for file in glob.glob('../../**/*.md', recursive=True):
        if len(file.split('/')) < 4 or file.endswith('README.md'):
            continue
        src = file
        dst = file.split('/')[2:]
        
        computed_dst = os.path.join('docs', 'templates', '/'.join(dst))
        
        os.makedirs(os.path.dirname(computed_dst), exist_ok=True)
        shutil.move(src, computed_dst)
        
        print(f'✅ {src} moved to {computed_dst}')

def create_category_readmes(notebooks):
    for category in notebooks:
        try:
            with open(os.path.join('docs', 'templates', category, 'README.md'), 'w') as readme:
                readme.write(f'# {category}')
            readme.close()
        except Exception as e:
            print(f'❌ Error while creating category [{category}] README.md')
            print(e)
            print('⚠️ Skipping this one.')<jupyter_output><empty_output><jupyter_text>Output<jupyter_code>got_errors = False
error_counter = 0

notebooks = {}

# Sort notebooks by technology (category)
for file in glob.glob('../../**/*.ipynb', recursive=True):
    # Do not check notebooks in .github or at the root of the project.
    if '.github' in file or len(file.split('/')) == 3:
        continue

    technology, action = get_technology_and_action(file)
    if technology not in notebooks:
        notebooks[technology] = []
    notebooks[technology].append((action, file))

# Create readme lines for all notebooks per technology.
readme_lines = create_readme_lines(notebooks)

# Remove titles and naas logo.
remove_unwanted_cells(notebooks)

# Convert .ipynb files to .md files.
convert_notebooks_to_md(notebooks)

# Create a new SUMMARY.md files updated with all notebooks.
with open('docs/new_SUMMARY.md', 'w') as new_summary:
    with open('docs/SUMMARY.md') as summary:
        for line in summary.readlines():
            if line == "* [😎 Templates](templates.md)\n":
                new_summary.write(line)
                for rl in readme_lines:
                    new_summary.write(rl)
            elif "(templates/" in line:
                continue
            else:
                new_summary.write(line)
        summary.close()
    new_summary.close()
shutil.move('docs/new_SUMMARY.md', 'docs/SUMMARY.md')


# Moving markdown files into documentations repo.
move_markdown_files()

# Create README.md for each category.
create_category_readmes(notebooks)

# Complete
print('✅ Process completed!')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14218645204560698, 'fraction_numerical': 0.004471272076905879, 'mean_word_length': 3.0053715308863027, 'pattern_counts': {'":': 1, '<': 19, '<?xml version=': 0, '>': 18, 'https://': 1, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '291143', 'license': 'BSD-3-Clause', 'n_tokens_mistral': 1456, 'n_tokens_neox': 1409, 'n_words': 306, 'path': '.github/documentation/documentation_update.ipynb', 'repo_name': 'srini047/awesome-notebooks', 'size': '4480'}