<commit_before># -*- coding: utf-8 -*-
import scrapy
import csv
import re
from datetime import datetime

class BibbSpider(scrapy.Spider):
    name = 'bibb'
    allowed_domains = ['co.bibb.ga.us']
    start_urls = ['http://www.co.bibb.ga.us/BSOInmatesOnline/CurrentDayMaster.asp']
    custom_settings = {
        'FEED_URI': 'output/bibb-output-%s.csv' % datetime.now(),
        'FEED_FORMAT': 'csv'
    }

    def parse(self, response):
        page = response.url.split("/")[-2]
        filename = 'bib-%s.html' % page

        for href in response.css('table tr > td > a::attr(href)'):
            yield response.follow(href, callback=self.parseRecord)

    def parseRecord(self, response):

        def formatXpathSelector(xpathSelector):
            return record.xpath("translate(normalize-space(\
                %s/td[2]/strong/text()), ' ', '')" % xpathSelector)

        def extract(xpathSelector):
            return formatXpathSelector(xpathSelector).extract_first()

        def extractExact(xpathSelector):
            return record.xpath(xpathSelector).extract_first()

        def formatInmateName(initalName):
            lastName = initalName.pop(0)
            # Remove comma
            initalName.pop(0)
            firstName = initalName.pop(1)
            middleName = ''.join(initalName).strip()
            return {
                'first': firstName,
                'middle': middleName,
                'last': lastName
            }

        def parseRaceSex(initalValue):
            value = initalValue.split('/')
            return {
                'race': value[0].strip(),
                'sex': value[1].strip()
            }

        for record in response.css('table'):
            inmateId = extract('./tr[2]')
            if (inmateId):

                inmateName = formatInmateName(
                    re.findall(r'\s|,|[^,\s]+', extract('./tr[3]')))<commit_msg>Rename variable for readability<commit_after># -*- coding: utf-8 -*-
import scrapy
import csv
import re
from datetime import datetime

class BibbSpider(scrapy.Spider):
    name = 'bibb'
    allowed_domains = ['co.bibb.ga.us']
    start_urls = ['http://www.co.bibb.ga.us/BSOInmatesOnline/CurrentDayMaster.asp']
    custom_settings = {
        'FEED_URI': 'output/bibb-output-%s.csv' % datetime.now(),
        'FEED_FORMAT': 'csv'
    }

    def parse(self, response):
        page = response.url.split("/")[-2]
        filename = 'bib-%s.html' % page

        for href in response.css('table tr > td > a::attr(href)'):
            yield response.follow(href, callback=self.parseRecord)

    def parseRecord(self, response):

        def formatXpathSelector(xpathSelector):
            return record.xpath("translate(normalize-space(\
                %s/td[2]/strong/text()), ' ', '')" % xpathSelector)

        def extract(xpathSelector):
            return formatXpathSelector(xpathSelector).extract_first()

        def extractExact(xpathSelector):
            return record.xpath(xpathSelector).extract_first()

        def formatInmateName(initalName):
            lastName = initalName.pop(0)
            # Remove comma
            initalName.pop(0)
            firstName = initalName.pop(1)
            middleName = ''.join(initalName).strip()
            return {
                'first': firstName,
                'middle': middleName,
                'last': lastName
            }

        def parseRaceSex(input):
            value = input.split('/')
            return {
                'race': value[0].strip(),
                'sex': value[1].strip()
            }

        for record in response.css('table'):
            inmateId = extract('./tr[2]')
            if (inmateId):

                inmateName = formatInmateName(
                    re.findall(r'\s|,|[^,\s]+', extract('./tr[3]')))
{'fraction_non_alphanumeric': 0.13948610382800208, 'fraction_numerical': 0.005243838489774515, 'mean_word_length': 1.98046875, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5896744', 'n_tokens_mistral': 1166, 'n_tokens_neox': 1101, 'n_words': 204}