<commit_before>import argparse
import os
from os.path import isdir, isfile


BASE_DIR = './tempcontent/pages/'
BASE_FSP = "https://www.fullstackpython.com/"

links = {
         "(/table-of-contents.html)":
         "(#table-of-contents)",

         # chapter 1
         "(/introduction.html)":
         "(#introduction)",
         "(/learning-programming.html)":
         "(#learning-programming)",
         "(/python-programming-language.html)":
         "(#python-programming-language)",
         "(/why-use-python.html)":
         "(#why-use-python)",
         "(/python-2-or-3.html)":
         "(#python-2-or-3)",
         "(/enterprise-python.html)":
         "(#enterprise-python)",
         "(/python-community.html)":
         "(#python-community)",
         "(/companies-using-python.html)":
         "(#companies-using-python)",
         "(/best-python-resources.html)":
         "(#best-python-resources)",
         "(/best-python-videos.html)":
         "(#best-python-videos)",
         "(/best-python-podcasts.html)":
         "(#best-python-podcasts)",

         # chapter 2
         "(/development-environments.html)":
         "(#development-environments)",
         "(/text-editors-ides.html)":
         "(#text-editors-and-ides)",
         "(/vim.html)":
         "(#vim)",
         "(/emacs.html)":
         "(#emacs)",
         "(/sublime-text.html)":
         "(#sublime-text)",
         "(/pycharm.html)":
         "(#pycharm)",
         "(/jupyter-notebook.html)":
         "(#jupyter-notebook)",
         "(/shells.html)":
         "(#shells)",
         "(/bourne-again-shell-bash.html)":
         "(#bourne-again-shell-bash)",
         "(/zsh-shell.html)":
         "(#zsh)",
         "(/powershell.html)":
         "(#powershell)",
         "(/terminal-multiplexers.html)":
         "(#terminal-multiplexers)",
         "(/tmux.html)":
         "(#tmux)",
         "(/screen.html)":
         "(#screen)",
         "(/environment-configuration.html)":
         "(#environment-configuration)",
         "(/application-dependencies.html)":
         "(#application-dependencies)",
         "(/virtual-environments-virtualenvs-venvs.html)":
         "(#virtual-environments-virtualenvs)",
         "(/localhost-tunnels.html)":
         "(#localhost-tunnels)",
         "(/source-control.html)":
         "(#source-control)",
         "(/git.html)":
         "(#git)",
         "(/mercurial.html)":
         "(#mercurial)",

         # chapter 3
         "(/data.html)":
         "(#data)",
         "(/databases.html)":
         "(#relational-databases)",
         "(/postgresql.html)":
         "(#postgresql)",
         "(/mysql.html)":
         "(#mysql)",
         "(/sqlite.html)":
         "(#sqlite)",
         "(/object-relational-mappers-orms.html)":
         "(#object-relational-mappers-orms)",
         "(/sqlalchemy.html)":
         "(#sqlalchemy)",
         "(/peewee.html)":
         "(#peewee)",
         "(/django-orm.html)":
         "(#django-object-relational-mapper)",
         "(/pony-orm.html)":
         "(#pony-orm)",
         "(/no-sql-datastore.html)":
         "(#nosql-data-stores)",
         "(/redis.html)":
         "(#redis)",
         "(/mongodb.html)":
         "(#mongodb)",
         "(/apache-cassandra.html)":
         "(#apache-cassandra)",
         "(/neo4j.html)":
         "(#neo4j)",
         "(/data-analysis.html)":<commit_msg>working on fixing broken links<commit_after>import argparse
import os
from os.path import isdir, isfile


BASE_DIR = './tempcontent/pages/'
BASE_FSP = "https://www.fullstackpython.com/"

links = {
         "(/table-of-contents.html)":
         "(#table-of-contents)",

         # chapter 1
         "(/introduction.html)":
         "(#introduction)",
         "(/learning-programming.html)":
         "(#learning-programming)",
         "(/python-programming-language.html)":
         "(#python-programming-language)",
         "(/why-use-python.html)":
         "(#why-use-python)",
         "(/python-2-or-3.html)":
         "(#python-2-or-3)",
         "(/enterprise-python.html)":
         "(#enterprise-python)",
         "(/python-community.html)":
         "(#python-community)",
         "(/companies-using-python.html)":
         "(#companies-using-python)",
         "(/best-python-resources.html)":
         "(#best-python-resources)",
         "(/best-python-videos.html)":
         "(#best-python-videos)",
         "(/best-python-podcasts.html)":
         "(#best-python-podcasts)",

         # chapter 2
         "(/development-environments.html)":
         "(#development-environments)",
         "(/text-editors-ides.html)":
         "(#text-editors-ides)",
         "(/vim.html)":
         "(#vim)",
         "(/emacs.html)":
         "(#emacs)",
         "(/sublime-text.html)":
         "(#sublime-text)",
         "(/pycharm.html)":
         "(#pycharm)",
         "(/jupyter-notebook.html)":
         "(#jupyter-notebook)",
         "(/shells.html)":
         "(#shells)",
         "(/bourne-again-shell-bash.html)":
         "(#bourne-again-shell-bash)",
         "(/zsh-shell.html)":
         "(#zsh-shell)",
         "(/powershell.html)":
         "(#powershell)",
         "(/terminal-multiplexers.html)":
         "(#terminal-multiplexers)",
         "(/tmux.html)":
         "(#tmux)",
         "(/screen.html)":
         "(#screen)",
         "(/environment-configuration.html)":
         "(#environment-configuration)",
         "(/application-dependencies.html)":
         "(#application-dependencies)",
         "(/virtual-environments-virtualenvs-venvs.html)":
         "(#virtual-environments-virtualenvs-venvs)",
         "(/localhost-tunnels.html)":
         "(#localhost-tunnels)",
         "(/source-control.html)":
         "(#source-control)",
         "(/git.html)":
         "(#git)",
         "(/mercurial.html)":
         "(#mercurial)",

         # chapter 3
         "(/data.html)":
         "(#data)",
         "(/databases.html)":
         "(#relational-databases)",
         "(/postgresql.html)":
         "(#postgresql)",
         "(/mysql.html)":
         "(#mysql)",
         "(/sqlite.html)":
         "(#sqlite)",
         "(/object-relational-mappers-orms.html)":
         "(#object-relational-mappers-orms)",
         "(/sqlalchemy.html)":
         "(#sqlalchemy)",
         "(/peewee.html)":
         "(#peewee)",
         "(/django-orm.html)":
         "(#django-orm)",
         "(/pony-orm.html)":
         "(#pony-orm)",
         "(/no-sql-datastore.html)":
         "(#nosql-data-store)",
         "(/redis.html)":
         "(#redis)",
         "(/mongodb.html)":
         "(#mongodb)",
         "(/apache-cassandra.html)":
         "(#apache-cassandra)",
         "(/neo4j.html)":
         "(#neo4j)",
         "(/data-analysis.html)":
{'fraction_non_alphanumeric': 0.22073332351641878, 'fraction_numerical': 0.002650566926814902, 'mean_word_length': 1.7564935064935066, 'pattern_counts': {'":': 98, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5855557', 'n_tokens_mistral': 2257, 'n_tokens_neox': 2174, 'n_words': 237}