<commit_before># -*- coding: utf-8 -*-
"""
Wafflehouse has 1906 locations and connects to a 3rd party end point
https://api.sweetiq.com/store-locator/public/locations/587d236eeb89fb17504336db.
I am guessing 587d236eeb89fb17504336db is what uniquely identifies wafflehouse to this API

The API works by returning locations within a bounding box and there is a perPage URL param
that allows us to specify how many results we need.
When skipped, it returns all.

Widening the box to cover the whole of the United states returned all
1906 results worth 252kb of JSON so we are safe returning all
"""
import scrapy
import json
import re
from locations.items import GeojsonPointItem


class WaffleHouseSpider(scrapy.Spider):
    name = "wafflehouse"
    allowed_domains = ["sweetiq.com"]
    start_urls = ('https://api.sweetiq.com/store-locator/public/locations/587d236eeb89fb17504336db'
                  '?categories=&geo%5B0%5D=-74.0468&geo%5B1%5D=40.8859&tag=&page=1&'
                  'search=&searchFields%5B0%5D=name&box%5B0%5D=-163.37843921359377&box%5B1%5D=-14.777946567390337&'
                  'box%5B2%5D=-44.198751713593765&box%5B3%5D=70.9012314513329&clientIds%5B0%5D=56fd9c824a88871f1d26062a',)

    def parse(self, response):
        locations = json.loads(response.text)
        if locations['records']:
            for location in locations['records']:
                full_address = location['address'] + " " + location['addressLine2']
                city = location['city']
                state = location['province']
                country = location['country']
                zip_code = location['postalCode']
                latitude = location['geo'][0]
                longitude = location['geo'][1]
                phone = location['phone']
                open_hours = location['hoursOfOperation']
                website = location['website']
                link_id = location['_id']

                properties = {"addr_full": full_address,
                              "city": city,
                              "state": state,
                              "postcode": zip_code,
                              "country": country,
                              "phone": self.process_phone(phone),
                              "website": website,
                              "ref": link_id,
                              "opening_hours": self.process_hours(open_hours),
                              "lon": float(longitude),
                              "lat": float(latitude)}
                yield GeojsonPointItem(**properties)

    def process_phone(self, phone_number):
        """
        converts phone number from (610) 838-2592 to 610-838-2592
        :param phone_number: the phone number to convert
        :return:
        """
        find_numbers = re.search(r"\((\d{3})\)\s(\d{3})-(\d{4})", phone_number)
        if find_numbers:
            num = find_numbers.groups()<commit_msg>Flip lat and lon around<commit_after># -*- coding: utf-8 -*-
"""
Wafflehouse has 1906 locations and connects to a 3rd party end point
https://api.sweetiq.com/store-locator/public/locations/587d236eeb89fb17504336db.
I am guessing 587d236eeb89fb17504336db is what uniquely identifies wafflehouse to this API

The API works by returning locations within a bounding box and there is a perPage URL param
that allows us to specify how many results we need.
When skipped, it returns all.

Widening the box to cover the whole of the United states returned all
1906 results worth 252kb of JSON so we are safe returning all
"""
import scrapy
import json
import re
from locations.items import GeojsonPointItem


class WaffleHouseSpider(scrapy.Spider):
    name = "wafflehouse"
    allowed_domains = ["sweetiq.com"]
    start_urls = ('https://api.sweetiq.com/store-locator/public/locations/587d236eeb89fb17504336db'
                  '?categories=&geo%5B0%5D=-74.0468&geo%5B1%5D=40.8859&tag=&page=1&'
                  'search=&searchFields%5B0%5D=name&box%5B0%5D=-163.37843921359377&box%5B1%5D=-14.777946567390337&'
                  'box%5B2%5D=-44.198751713593765&box%5B3%5D=70.9012314513329&clientIds%5B0%5D=56fd9c824a88871f1d26062a',)

    def parse(self, response):
        locations = json.loads(response.text)
        if locations['records']:
            for location in locations['records']:
                full_address = location['address'] + " " + location['addressLine2']
                city = location['city']
                state = location['province']
                country = location['country']
                zip_code = location['postalCode']
                latitude = location['geo'][1]
                longitude = location['geo'][0]
                phone = location['phone']
                open_hours = location['hoursOfOperation']
                website = location['website']
                link_id = location['_id']

                properties = {
                    "addr_full": full_address,
                    "city": city,
                    "state": state,
                    "postcode": zip_code,
                    "country": country,
                    "phone": self.process_phone(phone),
                    "website": website,
                    "ref": link_id,
                    "opening_hours": self.process_hours(open_hours),
                    "lon": longitude,
                    "lat": latitude,
                }

                yield GeojsonPointItem(**properties)

    def process_phone(self, phone_number):
        """
        converts phone number from (610) 838-2592 to 610-838-2592
        :param phone_number: the phone number to convert
        :return:
        """
        find_numbers = re.search(r"\((\d{3})\)\s(\d{3})-(\d{4})", phone_number)
        if find_numbers:
            num = find_numbers.groups()
{'fraction_non_alphanumeric': 0.10852311939268461, 'fraction_numerical': 0.07142857142857142, 'mean_word_length': 2.1166666666666667, 'pattern_counts': {'":': 22, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3495918', 'n_tokens_mistral': 1967, 'n_tokens_neox': 1641, 'n_words': 385}