<commit_before>
import sys
import logging
import os.path
import shutil
from lxml import etree, html
from clean.empty import elements
from clean.date import date_cleanser
from clean.markup import markup_cleanser
import requests

log = logging.getLogger('TRANSFORMER')

class Transformer:

    def __init__(self, files_list, site, output_folder, transforms, solr_service):
        # what to process
        self.files_list = files_list

        # the site code
        self.site = site

        # where to stash the output
        self.output_folder = output_folder
        if not os.path.isdir(self.output_folder):
            os.makedirs(self.output_folder)

        # how to process it
        self.transforms = transforms

        # the names of the fields which could have a date
        self.date_fields = [ 'date_from', 'date_to' ]

        # the names of the fields which could have markup
        self.markup_fields = [ 'abstract', 'text', 'locality' ]

        # get the date bounds from the existing dataset and 
        #  use those to set date bounds on entities with missing
        #  date contet: +/- 10
        resp = requests.get("%s/select?q=*:*&rows=1&wt=json&sort=date_from asc" % solr_service)
        resp = resp.json()
        date_lower_bound = resp['response']['docs'][0]['date_from'].split('-')[0]
        self.date_lower_bound = int(date_lower_bound) - int(date_lower_bound[-1:]) - 10

        resp = requests.get("%s/select?q=*:*&rows=1&wt=json&sort=date_to desc" % solr_service)
        resp = resp.json()
        self.date_upper_bound = int(resp['response']['docs'][0]['date_to'].split('-')[0]) + 10
        log.debug("Dataset date boundaries: %s - %s" % (self.date_lower_bound, self.date_upper_bound))

        log.info('Transformer initialised')

    def run(self):
        """Process each document in the input files_list"""

        # wipe the output folder
        shutil.rmtree(self.output_folder)
        if not os.path.isdir(self.output_folder):
            os.makedirs(self.output_folder)

        for f in self.files_list:
            self.process_document(f)

    def process_document(self, doc, debug=False):
        """The code to actually process the document<commit_msg>handle the case of an empty index<commit_after>
import sys
import logging
import os.path
import shutil
from lxml import etree, html
from clean.empty import elements
from clean.date import date_cleanser
from clean.markup import markup_cleanser
import requests

log = logging.getLogger('TRANSFORMER')

class Transformer:

    def __init__(self, files_list, site, output_folder, transforms, solr_service):
        # what to process
        self.files_list = files_list

        # the site code
        self.site = site

        # where to stash the output
        self.output_folder = output_folder
        if not os.path.isdir(self.output_folder):
            os.makedirs(self.output_folder)

        # how to process it
        self.transforms = transforms

        # the names of the fields which could have a date
        self.date_fields = [ 'date_from', 'date_to' ]

        # the names of the fields which could have markup
        self.markup_fields = [ 'abstract', 'text', 'locality' ]

        # get the date bounds from the existing dataset and 
        #  use those to set date bounds on entities with missing
        #  date contet: +/- 10
        try:
            resp = requests.get("%s/select?q=*:*&rows=1&wt=json&sort=date_from asc" % solr_service)
            resp = resp.json()
            date_lower_bound = resp['response']['docs'][0]['date_from'].split('-')[0]
            self.date_lower_bound = int(date_lower_bound) - int(date_lower_bound[-1:]) - 10
        except:
            # if the index is empty - set a stupid start date
            #  this will become more reasonable on the next run when data exists
            self.date_lower_bound = 0001

        try:
            resp = requests.get("%s/select?q=*:*&rows=1&wt=json&sort=date_to desc" % solr_service)
            resp = resp.json()
            self.date_upper_bound = int(resp['response']['docs'][0]['date_to'].split('-')[0]) + 10
            log.debug("Dataset date boundaries: %s - %s" % (self.date_lower_bound, self.date_upper_bound))
        except:
            # if the index is empty - set a stupid end date
            #  this will become more reasonable on the next run when data exists
            self.date_upper_bound = 9000

        log.info('Transformer initialised')

    def run(self):
        """Process each document in the input files_list"""

        # wipe the output folder
        shutil.rmtree(self.output_folder)
        if not os.path.isdir(self.output_folder):
            os.makedirs(self.output_folder)

        for f in self.files_list:
            self.process_document(f)

    def process_document(self, doc, debug=False):
        """The code to actually process the document
{'fraction_non_alphanumeric': 0.10195758564437195, 'fraction_numerical': 0.006933115823817292, 'mean_word_length': 2.614591009579956, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1512802', 'n_tokens_mistral': 1471, 'n_tokens_neox': 1422, 'n_words': 413}