<commit_before>import glob
import os
import re
import shutil
import subprocess
import sys
from contextlib import contextmanager

@contextmanager
def cd(dir):
    prevdir = os.getcwd()
    os.chdir(os.path.expanduser(dir))
    try:
        yield
    finally:
        os.chdir(prevdir)

# Global variables
## Formatting
class Colors:
    NOTICE = '\033[94m'
    OK = '\033[92m'
    ERROR = '\033[91m'
    END = '\033[0m'
    
if not os.path.isfile('./conf.py'):
  print("Could not find Sphinx's configuration file. Make sure to start this script within Mantella's documentation root path.")
  exit(1)

os.makedirs('./.tmp', exist_ok=True)
os.makedirs('./.examples', exist_ok=True)
os.makedirs('./.images', exist_ok=True)
os.makedirs('./.animations', exist_ok=True)

an_error_occured = False
changelog = []
# Finds all header files inside `../include`
# *Note:* This is only done separately to extract the provide a progress bar, wherefore the number of files needs to be
# known beforehand.
files = []
for file in glob.glob('../include/*/*.hpp'):
  files.append([file, os.path.join('./api_reference', *(file.replace('.hpp', '.rst').split(os.path.sep)[3:]))])
for file in glob.glob('../include/*/*/*.hpp'):
  files.append([file, os.path.join('./api_reference', *(file.replace('.hpp', '.include').split(os.path.sep)[3:]))])
  
# Iterates over each header and:
# 1. Checks whether the files contains an documentation block (i.e. `/** ... */`) or not (only documented files are 
#    further processed)
# 2. Looks for `.. code-block::[c++|image]` blocks
#    a) `c++` blocks are compiled and executed. Their output is appended (with its own `.. code-block::` block) to the  
#        code example.
#    b) `image` blocks are also compiles and executed. However, we also expect an `:octave:` delimiter, separating the 
#       data generation in C++ from the visualisation by Octave. The black is then replaced with the image by Octave,
#       appended by its data generation and visualisation (each in its own `.. code-block::` block).
for file in files:
  print(Colors.NOTICE + '[{:3d}%]'.format(int(100.0 * (files.index(file) + 1) / len(files))) + Colors.END + ' ' + os.path.basename(file[1]), end="", flush=True)

  headerfile = open(file[0], mode='r', encoding='utf-8')
  comments = ''.join(re.findall(r'[ ]*\/\*\*(.+?)(?=\*\/)', headerfile.read(), re.DOTALL))
  headerfile.close()

  if not comments: 
    an_error_occured = True
    print(' ' + Colors.ERROR + 'No comments found' + Colors.END)
    continue
  
  # Searched changelog tags in comments
  for changes in re.findall(r'\.\. cpp:[function|class]+::[ ]+([a-z_]+).*?\n[ ]*\n(.*?)(?:\n[ ]*\n|$)', comments, re.DOTALL):
    if not changes or not 'versionadded' in changes[1]:
      an_error_occured = True
      print(' ' + Colors.ERROR + 'No versionadded tag found' + Colors.END)
      break
    else:
      changelog = changelog + [(float(re.search(r'versionadded:: (\d.\d+).*', changes[1]).group(1)),2,changes[0])]
        
    if 'versionchanged' in changes[1]:
      for vchanges in re.findall(r'versionchanged:: (\d.\d+)\n[ ]+(.*)', comments):
        changelog = changelog + [(float(vchanges[0]),1,changes[0],vchanges[1])]
    
    if 'deprecated' in changes[1]:
        changelog = changelog + [(float(re.search(r'deprecated:: (\d.\d+).*', changes[1]).group(1)),0,changes[0])]

  # Adds column widths to list-table tags
  first_column  = 27
  comments = re.sub(r'(( +).. list-table:: .*?\n)',  '\\1\\2  :widths: ' + str(first_column) + ' ' + str(100 - first_column) + '\n', comments, 0, re.DOTALL)

  # Adds column widths to list-table tags
  first_column  = 27
  comments = re.sub(r'(( +).. list-table:: .*?\n)',  '\\1\\2  :widths: ' + str(first_column) + ' ' + str(100 - first_column) + '\n', comments, 0, re.DOTALL)

  # Create subdirectory if missing
  os.makedirs(os.path.dirname(file[1]), exist_ok=True)


  with open(file[1], mode='w', encoding='utf-8') as docfile:
    # for part in re.findall(r''
      # '(.*?)(?=(?:[ ]*\.\. code-block::|$))' # Matches anything until a `.. code-block::` occurs.
      # '('
        # '([ ]*)\.\. code-block:: ([^ ]+)' # Matches the type of `.. code-block::` (`c++`, `image` or anything else).
        # '(?:.*?(?=:name:):name: ([^ ]+)|)' # Skips anything before `:name:` occurs and matches the name.
        # '(?:\n\3  :[^\n]+)*' # Skips all further lines that have `.. code-block::` indentation but start with ':'.
        # '((?:\n\3  [^\n]+|\n[ ]*)+)\n|' # Matches all lines having `.. code-block::` indentation and empty lines.
      # ')', comments, re.DOTALL):
    for part in re.findall(r'(.*?)(?=(?:[ ]*\.\. code-block::|$))(([ ]*)\.\. code-block:: ([^ ]+)(?:.*?(?=:name:):name: ([^ ]+)|)(?:\n\3  :[^\n]+)*((?:\n\3  [^\n]+|\n[ ]*)+)\n|)', comments, re.DOTALL):
      docfile.write(part[0])

      if part[1]:
        # Processes C++ blocks
        if 'c++' in part[3]:
          if part[4]:
            example = open('./examples/' + part[4], mode='w+')
            example.write(part[5].replace('\n' + part[2] + '  ', '\n').strip(' \t\n\r'))
            example.close()
          
          with cd("./.tmp"):
            example = open('./example.cpp', mode='w+')
            example.write(part[5])
            example.close()<commit_msg>fix: Removed the duplicated :widths: addition (#352)<commit_after>import glob
import os
import re
import shutil
import subprocess
import sys
from contextlib import contextmanager

@contextmanager
def cd(dir):
    prevdir = os.getcwd()
    os.chdir(os.path.expanduser(dir))
    try:
        yield
    finally:
        os.chdir(prevdir)

# Global variables
## Formatting
class Colors:
    NOTICE = '\033[94m'
    OK = '\033[92m'
    ERROR = '\033[91m'
    END = '\033[0m'
    
if not os.path.isfile('./conf.py'):
  print("Could not find Sphinx's configuration file. Make sure to start this script within Mantella's documentation root path.")
  exit(1)

os.makedirs('./.tmp', exist_ok=True)
os.makedirs('./.examples', exist_ok=True)
os.makedirs('./.images', exist_ok=True)
os.makedirs('./.animations', exist_ok=True)

an_error_occured = False
changelog = []
# Finds all header files inside `../include`
# *Note:* This is only done separately to extract the provide a progress bar, wherefore the number of files needs to be
# known beforehand.
files = []
for file in glob.glob('../include/*/*.hpp'):
  files.append([file, os.path.join('./api_reference', *(file.replace('.hpp', '.rst').split(os.path.sep)[3:]))])
for file in glob.glob('../include/*/*/*.hpp'):
  files.append([file, os.path.join('./api_reference', *(file.replace('.hpp', '.include').split(os.path.sep)[3:]))])
  
# Iterates over each header and:
# 1. Checks whether the files contains an documentation block (i.e. `/** ... */`) or not (only documented files are 
#    further processed)
# 2. Looks for `.. code-block::[c++|image]` blocks
#    a) `c++` blocks are compiled and executed. Their output is appended (with its own `.. code-block::` block) to the  
#        code example.
#    b) `image` blocks are also compiles and executed. However, we also expect an `:octave:` delimiter, separating the 
#       data generation in C++ from the visualisation by Octave. The black is then replaced with the image by Octave,
#       appended by its data generation and visualisation (each in its own `.. code-block::` block).
for file in files:
  print(Colors.NOTICE + '[{:3d}%]'.format(int(100.0 * (files.index(file) + 1) / len(files))) + Colors.END + ' ' + os.path.basename(file[1]), end="", flush=True)

  headerfile = open(file[0], mode='r', encoding='utf-8')
  comments = ''.join(re.findall(r'[ ]*\/\*\*(.+?)(?=\*\/)', headerfile.read(), re.DOTALL))
  headerfile.close()

  if not comments: 
    an_error_occured = True
    print(' ' + Colors.ERROR + 'No comments found' + Colors.END)
    continue
  
  # Searched changelog tags in comments
  for changes in re.findall(r'\.\. cpp:[function|class]+::[ ]+([a-z_]+).*?\n[ ]*\n(.*?)(?:\n[ ]*\n|$)', comments, re.DOTALL):
    if not changes or not 'versionadded' in changes[1]:
      an_error_occured = True
      print(' ' + Colors.ERROR + 'No versionadded tag found' + Colors.END)
      break
    else:
      changelog = changelog + [(float(re.search(r'versionadded:: (\d.\d+).*', changes[1]).group(1)),2,changes[0])]
        
    if 'versionchanged' in changes[1]:
      for vchanges in re.findall(r'versionchanged:: (\d.\d+)\n[ ]+(.*)', comments):
        changelog = changelog + [(float(vchanges[0]),1,changes[0],vchanges[1])]
    
    if 'deprecated' in changes[1]:
        changelog = changelog + [(float(re.search(r'deprecated:: (\d.\d+).*', changes[1]).group(1)),0,changes[0])]

  # Adds column widths to list-table tags
  first_column = 27
  comments = re.sub(r'(( +).. list-table:: .*?\n)',  '\\1\\2  :widths: ' + str(first_column) + ' ' + str(100 - first_column) + '\n', comments, 0, re.DOTALL)

  # Create subdirectory if missing
  os.makedirs(os.path.dirname(file[1]), exist_ok=True)


  with open(file[1], mode='w', encoding='utf-8') as docfile:
    # for part in re.findall(r''
      # '(.*?)(?=(?:[ ]*\.\. code-block::|$))' # Matches anything until a `.. code-block::` occurs.
      # '('
        # '([ ]*)\.\. code-block:: ([^ ]+)' # Matches the type of `.. code-block::` (`c++`, `image` or anything else).
        # '(?:.*?(?=:name:):name: ([^ ]+)|)' # Skips anything before `:name:` occurs and matches the name.
        # '(?:\n\3  :[^\n]+)*' # Skips all further lines that have `.. code-block::` indentation but start with ':'.
        # '((?:\n\3  [^\n]+|\n[ ]*)+)\n|' # Matches all lines having `.. code-block::` indentation and empty lines.
      # ')', comments, re.DOTALL):
    for part in re.findall(r'(.*?)(?=(?:[ ]*\.\. code-block::|$))(([ ]*)\.\. code-block:: ([^ ]+)(?:.*?(?=:name:):name: ([^ ]+)|)(?:\n\3  :[^\n]+)*((?:\n\3  [^\n]+|\n[ ]*)+)\n|)', comments, re.DOTALL):
      docfile.write(part[0])

      if part[1]:
        # Processes C++ blocks
        if 'c++' in part[3]:
          if part[4]:
            example = open('./examples/' + part[4], mode='w+')
            example.write(part[5].replace('\n' + part[2] + '  ', '\n').strip(' \t\n\r'))
            example.close()
          
          with cd("./.tmp"):
            example = open('./example.cpp', mode='w+')
            example.write(part[5])
            example.close()
{'fraction_non_alphanumeric': 0.2150391948127359, 'fraction_numerical': 0.014806929255782444, 'mean_word_length': 3.0991669972233242, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7112189', 'n_tokens_mistral': 3708, 'n_tokens_neox': 3612, 'n_words': 910}