<commit_before>#The preprocessor role is to create a dictionary of book objects and 
#a number representing their order of appearance in the document.
#The number is used as hyperlink text for cross referencing.
#The preprocessor also edit the html page to insert correct chapter and section numbers. 

__author__ = 'efouh'

import os
import sys
import re
import datetime
import shutil
import subprocess
import fnmatch
import json
import config
from optparse import OptionParser
from operator import itemgetter, attrgetter
from xml.dom.minidom import parse, parseString
from string import whitespace as ws


#a method to detect if the topic element is an example
def isExample(topic):
    if 'example' in topic.lower() and topic.startswith('.. topic::'):
        return True
    else:
        return False

def isTable(topic):
    if 'table' in topic.lower() and topic.startswith('.. topic::'):
        return True
    else:
        return False


def isTheorem(topic):
    if 'theorem' in topic.lower() and topic.startswith('.. topic::'):
        return True
    else:
        return False


#defines the color of output text (warnings, errors, and info)
class bcolors:
    HEADER = '\033[95m'
    OKBLUE = '\033[94m'
    OKGREEN = '\033[92m'
    WARNING = '\033[93m'
    FAIL = '\033[91m'
    ENDC = '\033[0m'

    def disable(self):
        self.HEADER = ''
        self.OKBLUE = ''
        self.OKGREEN = ''
        self.WARNING = ''
        self.FAIL = ''
        self.ENDC = ''


#a class representing modules. Object attributes are module metadata.
#The constructor parses module file to extract metadata indentified by keywords
class modPreReq:

   def __init__(self,filename):
      self.name =''  #field short_display_name in the database
      self.description=''   #field name in the database
      self.prereq=[]    # field prerequisites in the database
      self.prereqNum =0
      self.short_display_name=''
      self.v_position=0
      self.h_position=0
      self.seconds_per_fast_problem=5.0
      self.live='true'
      self.summative='false'
      self.author = ''
      self.raw_html=''
      self.creation_date = ''
      self.last_modified =''
      self.covers=''
      self.exercises = []  #list of exercises in the module.
      p = re.compile('(%s)' % ('|'.join([c for c in ws])))
      flag = 0
      cpt=0
      start=-1
      end=-1
      len_wthsp=-1
      type=''
      desc=''
      fig = 1
      tab = 1
      exp = 1
      thr = 1 
      label = ''
      fls = open(filename,'r')
      data = fls.readlines()
      new_data = []
      fls.close()
      self.name = os.path.basename(filename)
      now = datetime.datetime.now()
      self.creation_date = now.strftime("%Y-%m-%d %H:%M:%S")
      self.last_modified = now.strftime("%Y-%m-%d %H:%M:%S")
      for line in data:
         cpt=cpt+1
         if ':short_name:' in line:
            str =  re.split('short_name:', line, re.IGNORECASE)[1]
            self.description = p.sub('',str.replace(' ',''))
         if ':prerequisites:' in line:
            str =  re.split('prerequisites:', line, re.IGNORECASE)[1]
            self.prereq = p.sub('',str).split(',')
         if ':author:' in line:
            str =  re.split('author:', line, re.IGNORECASE)[1]
            self.author = p.sub('',str.replace(' ','_')).replace('_',' ')
         if ':exercises:' in line:
            str =  re.split('exercises:', line, re.IGNORECASE)[1]
            self.exercises = p.sub('',str).split(',')
         if ':topic:' in line:
            str =  re.split('topic:', line, re.IGNORECASE)[1]
            self.covers =  p.sub('',str).split(',')
         #label = ''
         if line.startswith('.. _'):
            label =  re.split(':', re.split('.. _', line, re.IGNORECASE)[1], re.IGNORECASE)[0]
            if data[cpt+1].startswith('.. figure::') or data[cpt+1].startswith('.. odsafig::') or data[cpt+1].startswith('.. inlineav::'): 
               if os.path.splitext(os.path.basename(filename))[0] in config.table:
                 tb = config.table[os.path.splitext(os.path.basename(filename))[0]]
                 config.table[label] = tb + '.%s#' %fig
                 fig+=1
            if data[cpt+1].startswith('.. table::') or data[cpt+1].startswith('.. odsatab') or isTable(data[cpt+1]):
                if os.path.splitext(os.path.basename(filename))[0] in config.table:
                  tb = config.table[os.path.splitext(os.path.basename(filename))[0]]
                  config.table[label] = tb + '.%s#' %tab
                  tab+=1
                label = '-1' 
            if isExample(data[cpt+1]):
                if os.path.splitext(os.path.basename(filename))[0] in config.table:
                   tb = config.table[os.path.splitext(os.path.basename(filename))[0]]
                   config.table[label] = tb + '.%s#' %exp
                exp+=1
                label = '-2'
            if isTheorem(data[cpt+1]):
                if os.path.splitext(os.path.basename(filename))[0] in config.table:
                   tb = config.table[os.path.splitext(os.path.basename(filename))[0]]
                   config.table[label] = tb + '.%s#' %thr
                   thr+=1
                label = '-3'
 
         if isExample(line):
             if label == '-2':
                 label = ''
             else:
                 exp+=1
                    
         if isTheorem(line):
             if label == '-3':
                 label = ''
             else:
                 thr+=1
   

         if ':target:' in line:
             trgt =  re.split('target:', line, re.IGNORECASE)[1]
             trgt=p.sub('',trgt)
             if os.path.splitext(os.path.basename(filename))[0] in config.table:
                 tb = config.table[os.path.splitext(os.path.basename(filename))[0]]
                 config.table[trgt] = tb + '.%s#' %fig
                 fig+=1 
 
         if ('.. TODO::' in line  or '.. todo::' in line) and len_wthsp==-1 and start==-1 and end==-1:
            start = cpt+1
         if start==cpt:
            if line.isspace():
               start = cpt+1
            else:
               expr = re.match(r"^:\w+",line.replace(' ',''))
               if expr is not None:<commit_msg>Fixed bug that was requiring all slideshows diagram to have an id in order for the numbering to be correct.<commit_after>#The preprocessor role is to create a dictionary of book objects and 
#a number representing their order of appearance in the document.
#The number is used as hyperlink text for cross referencing.
#The preprocessor also edit the html page to insert correct chapter and section numbers. 

__author__ = 'efouh'

import os
import sys
import re
import datetime
import shutil
import subprocess
import fnmatch
import json
import config
from optparse import OptionParser
from operator import itemgetter, attrgetter
from xml.dom.minidom import parse, parseString
from string import whitespace as ws


#a method to detect if the topic element is an example
def isExample(topic):
    if 'example' in topic.lower() and topic.startswith('.. topic::'):
        return True
    else:
        return False

def isTable(topic):
    if 'table' in topic.lower() and topic.startswith('.. topic::'):
        return True
    else:
        return False


def isTheorem(topic):
    if 'theorem' in topic.lower() and topic.startswith('.. topic::'):
        return True
    else:
        return False


def isFigure(item):
    if item.startswith('.. figure::') or item.startswith('.. odsafig::') or item.startswith('.. inlineav::'):
        if item.startswith('.. inlineav::') and 'dgm' not in item:
            return False
        return True
    else:
        return False


#defines the color of output text (warnings, errors, and info)
class bcolors:
    HEADER = '\033[95m'
    OKBLUE = '\033[94m'
    OKGREEN = '\033[92m'
    WARNING = '\033[93m'
    FAIL = '\033[91m'
    ENDC = '\033[0m'

    def disable(self):
        self.HEADER = ''
        self.OKBLUE = ''
        self.OKGREEN = ''
        self.WARNING = ''
        self.FAIL = ''
        self.ENDC = ''


#a class representing modules. Object attributes are module metadata.
#The constructor parses module file to extract metadata indentified by keywords
class modPreReq:

   def __init__(self,filename):
      self.name =''  #field short_display_name in the database
      self.description=''   #field name in the database
      self.prereq=[]    # field prerequisites in the database
      self.prereqNum =0
      self.short_display_name=''
      self.v_position=0
      self.h_position=0
      self.seconds_per_fast_problem=5.0
      self.live='true'
      self.summative='false'
      self.author = ''
      self.raw_html=''
      self.creation_date = ''
      self.last_modified =''
      self.covers=''
      self.exercises = []  #list of exercises in the module.
      p = re.compile('(%s)' % ('|'.join([c for c in ws])))
      flag = 0
      cpt=0
      start=-1
      end=-1
      len_wthsp=-1
      type=''
      desc=''
      fig = 1
      tab = 1
      exp = 1
      thr = 1 
      label = ''
      anonym_fig = 0
      fls = open(filename,'r')
      data = fls.readlines()
      new_data = []
      fls.close()
      self.name = os.path.basename(filename)
      now = datetime.datetime.now()
      self.creation_date = now.strftime("%Y-%m-%d %H:%M:%S")
      self.last_modified = now.strftime("%Y-%m-%d %H:%M:%S")
      for line in data:
         cpt=cpt+1
         if ':short_name:' in line:
            str =  re.split('short_name:', line, re.IGNORECASE)[1]
            self.description = p.sub('',str.replace(' ',''))
         if ':prerequisites:' in line:
            str =  re.split('prerequisites:', line, re.IGNORECASE)[1]
            self.prereq = p.sub('',str).split(',')
         if ':author:' in line:
            str =  re.split('author:', line, re.IGNORECASE)[1]
            self.author = p.sub('',str.replace(' ','_')).replace('_',' ')
         if ':exercises:' in line:
            str =  re.split('exercises:', line, re.IGNORECASE)[1]
            self.exercises = p.sub('',str).split(',')
         if ':topic:' in line:
            str =  re.split('topic:', line, re.IGNORECASE)[1]
            self.covers =  p.sub('',str).split(',')
         #label = ''
         if line.startswith('.. _'):
            label =  re.split(':', re.split('.. _', line, re.IGNORECASE)[1], re.IGNORECASE)[0]
            if data[cpt+1].startswith('.. figure::') or data[cpt+1].startswith('.. odsafig::') or data[cpt+1].startswith('.. inlineav::'): 
               if os.path.splitext(os.path.basename(filename))[0] in config.table:
                 tb = config.table[os.path.splitext(os.path.basename(filename))[0]]
                 config.table[label] = tb + '.%s#' %fig
                 fig+=1
               label = 'label_used'
            if data[cpt+1].startswith('.. table::') or data[cpt+1].startswith('.. odsatab') or isTable(data[cpt+1]):
                if os.path.splitext(os.path.basename(filename))[0] in config.table:
                  tb = config.table[os.path.splitext(os.path.basename(filename))[0]]
                  config.table[label] = tb + '.%s#' %tab
                  tab+=1
                label = '-1' 
            if isExample(data[cpt+1]):
                if os.path.splitext(os.path.basename(filename))[0] in config.table:
                   tb = config.table[os.path.splitext(os.path.basename(filename))[0]]
                   config.table[label] = tb + '.%s#' %exp
                exp+=1
                label = '-2'
            if isTheorem(data[cpt+1]):
                if os.path.splitext(os.path.basename(filename))[0] in config.table:
                   tb = config.table[os.path.splitext(os.path.basename(filename))[0]]
                   config.table[label] = tb + '.%s#' %thr
                   thr+=1
                label = '-3'
 
         if isExample(line):
             if label == '-2':
                 label = ''
             else:
                 exp+=1
                    
         if isTheorem(line):
             if label == '-3':
                 label = ''
             else:
                 thr+=1
   
         if isFigure(line):
             if label != 'label_used':    
                 fig_label = 'anon_fig%s' %anonym_fig
                 config.table[fig_label] = fig 
                 fig+=1
                 anonym_fig+=1

         if ':target:' in line:
             trgt =  re.split('target:', line, re.IGNORECASE)[1]
             trgt=p.sub('',trgt)
             if os.path.splitext(os.path.basename(filename))[0] in config.table:
                 tb = config.table[os.path.splitext(os.path.basename(filename))[0]]
                 config.table[trgt] = tb + '.%s#' %fig
                 fig+=1 
 
         if ('.. TODO::' in line  or '.. todo::' in line) and len_wthsp==-1 and start==-1 and end==-1:
            start = cpt+1
         if start==cpt:
            if line.isspace():
               start = cpt+1
            else:
               expr = re.match(r"^:\w+",line.replace(' ',''))
               if expr is not None:
{'fraction_non_alphanumeric': 0.14473684210526316, 'fraction_numerical': 0.013540391676866586, 'mean_word_length': 1.9109329770652417, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1831670', 'n_tokens_mistral': 4276, 'n_tokens_neox': 4091, 'n_words': 1010}