<commit_before>import asyncio
import json
import logging
import math
import os
import sys
from argparse import ArgumentParser
from csv import DictWriter
from configparser import RawConfigParser
from datetime import date
from io import StringIO
from itertools import chain
from pathlib import Path
from urllib.parse import urlencode

import aiofiles
import pandas as pd
import numpy as np
from aiohttp import request
from geopy.distance import vincenty


DTYPE = dict(cnpj=np.str)
LOG_FORMAT = '[%(levelname)s] %(asctime)s: %(message)s'

logging.basicConfig(stream=sys.stdout, level=logging.INFO, format=LOG_FORMAT)


class SexPlacesNearBy:

    BASE_URL = 'https://maps.googleapis.com/maps/api/place/'
    KEYWORDS = ('acompanhantes',
                'adult entertainment club',
                'adult entertainment store',
                'gay sauna',
                'massagem',
                'modeling agency',
                'motel',
                'night club',
                'sex club',
                'sex shop',
                'strip club',
                'swinger clubs')

    def __init__(self, company, key=None):
        """
        :param company: (dict) Company with name, cnpj, latitude and longitude
        :param key: (str) Google Places API key
        """
        settings = RawConfigParser()
        settings.read('config.ini')
        self.company = company
        self.key = key or settings.get('Google', 'APIKey')
        self.latitude = self.company['latitude']
        self.longitude = self.company['longitude']
        self.places = []
        self.closest = None

    @property
    def company_name(self):
        return self.company.get('trade_name') or self.company.get('name')

    async def get_all_places(self):
        """
        Use aiohttp to get the closest place to the company according to each
        keyworn. Returns `None`, the results are saved on self.places.
        """
        tasks = [self.load_place(k) for k in self.KEYWORDS]
        await asyncio.gather(*tasks)

    async def get_closest(self):
        """
        Gets the closest place find in reads self.places, queries for its
        details and returns a dict with all the info on that place.
        """
        if not self.is_valid():
            return None

        await self.get_all_places()
        ordered = sorted(self.places, key=lambda x: x['distance'])

        for place in ordered:
            place = await self.load_details(place)
            name = place.get('name', '').lower()

            if place['keyword'] == 'motel' and 'hotel' in name:
                pass  # google returns hotels when looking for a motel

            else:
                prefix = '💋 ' if place['distance'] < 5 else ''
                msg = '{}Found something interesting {:.2f}m away from {}…'
                args = (prefix, place.get('distance'), self.company_name)
                logging.info(msg.format(*args))
                self.closest = place
                return place

    @staticmethod
    def is_valid_coordinate(coord):
        try:
            as_float = float(coord)
        except ValueError:
            return False

        return False if math.isnan(as_float) else True

    def is_valid(self):
        coords = (self.latitude, self.longitude)
        if not all(map(self.is_valid_coordinate, coords)):
            msg = 'No geolocation information for company: {} ({})'
            logging.info(msg.format(self.company_name, self.company['cnpj']))
            return False

        return True

    async def load_place(self, keyword):
        """
        Given a keyword it loads the place returned by the API to self.places.
        """
        keyword, content = await self.get(keyword)
        place = self.parse(keyword, content)
        if place and isinstance(place.get('distance'), float):
            self.places.append(place)

    async def get(self, keyword, print_status=False):
        if print_status:
            msg = 'Looking for a {} near {} ({})…'
            args = (keyword, self.company_name, self.company.get('cnpj'))
            logging.info(msg.format(*args))

        url = self.nearby_url(keyword)
        response = await request('GET', url)
        content = await response.text()
        return keyword, content

    def parse(self, keyword, content):
        """
        Return a dictonary with information of the nearest sex place
        around a given company.
        :param keyword: (str) keyword used by the request
        :param content: (str) content of the response to the request
        :return: (dict) with
            * name : The name of nearest sex place
            * latitude : The latitude of nearest sex place
            * longitude : The longitude of nearest sex place
            * distance : Distance (in meters) between the company and the
              nearest sex place
            * address : The address of the nearest sex place
            * phone : The phone of the nearest sex place
            * id : The Google Place ID of the nearest sex place
            * keyword : term that matched the sex place in Google Place Search
        """
        response = json.loads(content)
        if response['status'] != 'OK':
            if 'error_message' in response:
                status, error = response.get('status'), response.get('error')
                logging.info('{}: {}'.format(status, error))
                return None
            elif response.get('status') != 'ZERO_RESULTS':
                msg = 'Google Places API Status: {}'
                logging.info(msg.format(response.get('status')))
                return None

        else:
            place = response.get('results', [{}])[0]

            location = place.get('geometry', {}).get('location', {})
            latitude = float(location.get('lat'))
            longitude = float(location.get('lng'))

            company_location = (self.latitude, self.longitude)
            place_location = (latitude, longitude)
            distance = vincenty(company_location, place_location)
<commit_msg>Document Google Places API response status<commit_after>import asyncio
import json
import logging
import math
import os
import sys
from argparse import ArgumentParser
from csv import DictWriter
from configparser import RawConfigParser
from datetime import date
from io import StringIO
from itertools import chain
from pathlib import Path
from urllib.parse import urlencode

import aiofiles
import pandas as pd
import numpy as np
from aiohttp import request
from geopy.distance import vincenty


DTYPE = dict(cnpj=np.str)
LOG_FORMAT = '[%(levelname)s] %(asctime)s: %(message)s'

logging.basicConfig(stream=sys.stdout, level=logging.INFO, format=LOG_FORMAT)


class SexPlacesNearBy:

    BASE_URL = 'https://maps.googleapis.com/maps/api/place/'
    KEYWORDS = ('acompanhantes',
                'adult entertainment club',
                'adult entertainment store',
                'gay sauna',
                'massagem',
                'modeling agency',
                'motel',
                'night club',
                'sex club',
                'sex shop',
                'strip club',
                'swinger clubs')

    def __init__(self, company, key=None):
        """
        :param company: (dict) Company with name, cnpj, latitude and longitude
        :param key: (str) Google Places API key
        """
        settings = RawConfigParser()
        settings.read('config.ini')
        self.company = company
        self.key = key or settings.get('Google', 'APIKey')
        self.latitude = self.company['latitude']
        self.longitude = self.company['longitude']
        self.places = []
        self.closest = None

    @property
    def company_name(self):
        return self.company.get('trade_name') or self.company.get('name')

    async def get_all_places(self):
        """
        Use aiohttp to get the closest place to the company according to each
        keyworn. Returns `None`, the results are saved on self.places.
        """
        tasks = [self.load_place(k) for k in self.KEYWORDS]
        await asyncio.gather(*tasks)

    async def get_closest(self):
        """
        Gets the closest place find in reads self.places, queries for its
        details and returns a dict with all the info on that place.
        """
        if not self.is_valid():
            return None

        await self.get_all_places()
        ordered = sorted(self.places, key=lambda x: x['distance'])

        for place in ordered:
            place = await self.load_details(place)
            name = place.get('name', '').lower()

            if place['keyword'] == 'motel' and 'hotel' in name:
                pass  # google returns hotels when looking for a motel

            else:
                prefix = '💋 ' if place['distance'] < 5 else ''
                msg = '{}Found something interesting {:.2f}m away from {}…'
                args = (prefix, place.get('distance'), self.company_name)
                logging.info(msg.format(*args))
                self.closest = place
                return place

    @staticmethod
    def is_valid_coordinate(coord):
        try:
            as_float = float(coord)
        except ValueError:
            return False

        return False if math.isnan(as_float) else True

    def is_valid(self):
        coords = (self.latitude, self.longitude)
        if not all(map(self.is_valid_coordinate, coords)):
            msg = 'No geolocation information for company: {} ({})'
            logging.info(msg.format(self.company_name, self.company['cnpj']))
            return False

        return True

    async def load_place(self, keyword):
        """
        Given a keyword it loads the place returned by the API to self.places.
        """
        keyword, content = await self.get(keyword)
        place = self.parse(keyword, content)
        if place and isinstance(place.get('distance'), float):
            self.places.append(place)

    async def get(self, keyword, print_status=False):
        if print_status:
            msg = 'Looking for a {} near {} ({})…'
            args = (keyword, self.company_name, self.company.get('cnpj'))
            logging.info(msg.format(*args))

        url = self.nearby_url(keyword)
        response = await request('GET', url)
        content = await response.text()
        return keyword, content

    def parse(self, keyword, content):
        """
        Return a dictonary with information of the nearest sex place
        around a given company.
        :param keyword: (str) keyword used by the request
        :param content: (str) content of the response to the request
        :return: (dict) with
            * name : The name of nearest sex place
            * latitude : The latitude of nearest sex place
            * longitude : The longitude of nearest sex place
            * distance : Distance (in meters) between the company and the
              nearest sex place
            * address : The address of the nearest sex place
            * phone : The phone of the nearest sex place
            * id : The Google Place ID of the nearest sex place
            * keyword : term that matched the sex place in Google Place Search

        Google responses:
            * `OK` indicates that no errors occurred; the place was
              successfully detected and at least one result was returned.
            * `UNKNOWN_ERROR` indicates a server-side error; trying again may
              be successful.
            * `ZERO_RESULTS` indicates that the reference was valid but no
              longer refers to a valid result. This may occur if the
              establishment is no longer in business.
            * `OVER_QUERY_LIMIT` indicates that you are over your quota.
            * `REQUEST_DENIED` indicates that your request was denied,
              generally because of lack of an invalid key parameter.
            * `INVALID_REQUEST` generally indicates that the query (reference)
              is missing.
            * `NOT_FOUND` indicates that the referenced location was not found
              in the Places database.<Paste>

        Source: https://developers.google.com/places/web-service/details
        """
        response = json.loads(content)
        if response['status'] != 'OK':
            if 'error_message' in response:
                status, error = response.get('status'), response.get('error')
                logging.info('{}: {}'.format(status, error))
                return None
            elif response.get('status') != 'ZERO_RESULTS':
                msg = 'Google Places API Status: {}'
                logging.info(msg.format(response.get('status')))
                return None

        else:
            place = response.get('results', [{}])[0]

            location = place.get('geometry', {}).get('location', {})
            latitude = float(location.get('lat'))
            longitude = float(location.get('lng'))

            company_location = (self.latitude, self.longitude)
            place_location = (latitude, longitude)
            distance = vincenty(company_location, place_location)

{'fraction_non_alphanumeric': 0.09578107183580388, 'fraction_numerical': 0.00045610034207525655, 'mean_word_length': 2.0810304449648713, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 4, 'https://': 3, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 18, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7624434', 'n_tokens_mistral': 3500, 'n_tokens_neox': 3350, 'n_words': 1149}