<commit_before>#!/usr/bin/env python
#
# Copyright 2015 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

try:
    from google.appengine.api import taskqueue, urlfetch, app_identity
    import models
except Exception as e:
    if __name__ != '__main__':
        raise e
    else:
        print "Warning: import exception '{0}'".format(e)

from urllib import quote_plus
from urlparse import urljoin, urlparse
import cgi
import datetime
import json
import logging
import lxml.etree

################################################################################

ENABLE_EXPERIMENTAL = app_identity.get_application_id().endswith('-dev')
PHYSICAL_WEB_USER_AGENT = 'Mozilla/5.0' # TODO: Find a more descriptive string.
BASE_URL = 'https://' + app_identity.get_application_id() + '.appspot.com'

################################################################################

def BuildResponse(objects):
    metadata_output = []
    unresolved_output = []

    # Resolve the devices
    for obj in objects:
        url = obj.get('url', None)
        force_update = obj.get('force', False)
        rssi = obj.get('rssi', None)
        txpower = obj.get('txpower', None)
        distance = ComputeDistance(rssi, txpower)

        def append_invalid():
            #unresolved_output.append({
            #    'id': url
            #})
            pass

        if url is None:
            continue

        parsed_url = urlparse(url)
        if parsed_url.scheme != 'http' and parsed_url.scheme != 'https':
            append_invalid()
            continue

        try:
            siteInfo = GetSiteInfoForUrl(url, distance, force_update)
        except FailedFetchException:
            append_invalid()
            continue

        if siteInfo is None:
            # It's a valid url, which we didn't fail to fetch, so it must be `No Content`
            continue

        device_data = {}
        device_data['id'] = url
        # TODO: change url to the original url (perhaps minus our goo.gl shortened values)
        device_data['url'] = siteInfo.url
        # TODO: change displayUrl to the "most applicable" url (resolve shorteners, but perhaps not all redirects)
        device_data['displayUrl'] = siteInfo.url
        if siteInfo.title is not None:
            device_data['title'] = siteInfo.title
        if siteInfo.description is not None:
            device_data['description'] = siteInfo.description
        if siteInfo.favicon_url is not None:
            device_data['icon'] = urljoin(BASE_URL, '/favicon?url=' + quote_plus(siteInfo.favicon_url))
        if siteInfo.jsonlds is not None:
            device_data['json-ld'] = json.loads(siteInfo.jsonlds)
        device_data['distance'] = distance
        metadata_output.append(device_data)

    metadata_output = map(ReplaceDistanceWithRank, RankedResponse(metadata_output))

    ret = {
        "metadata": metadata_output,
    }

    if unresolved_output:
        ret["unresolved"] = unresolved_output

    return ret

################################################################################

def ComputeDistance(rssi, txpower):
    try:
        rssi = float(rssi)
        txpower = float(txpower)
        if rssi in [127, 128]: # Known invalid rssi values
            return None
        path_loss = txpower - rssi
        distance = pow(10.0, (path_loss - 41) / 20)
        return distance
    except:
        return None

def RankedResponse(metadata_output):
    def SortByDistanceCmp(a, b):
        return cmp(a['distance'], b['distance'])

    metadata_output.sort(SortByDistanceCmp)
    return metadata_output

def ReplaceDistanceWithRank(device_data):
    distance = device_data['distance']
    distance = distance if distance is not None else 1000
    device_data['rank'] = distance
    device_data.pop('distance', None)
    return device_data

################################################################################

# This is used to recursively look up in cache after each redirection.
# We don't cache the redirection itself, but we always want to cache the final destination.
def GetSiteInfoForUrl(url, distance=None, force_update=False):
    logging.info('GetSiteInfoForUrl url:{0}, distance:{1}'.format(url, distance))

    siteInfo = None

    if force_update:
        siteInfo = FetchAndStoreUrl(siteInfo, url, distance, force_update)
    else:
        siteInfo = models.SiteInformation.get_by_id(url)

        if siteInfo is None:
            siteInfo = FetchAndStoreUrl(siteInfo, url, distance, force_update)
        else:
            # If the cache is older than 5 minutes, queue a refresh
            updated_ago = datetime.datetime.now() - siteInfo.updated_on
            if updated_ago > datetime.timedelta(minutes=5):
                logging.info('Queue RefreshUrl for url: {0}, which was updated {1} ago'.format(url, updated_ago))
                # Add request to queue.
                taskqueue.add(url='/refresh-url', params={'url': url})

    return siteInfo

################################################################################<commit_msg>Add groupid to PWS response<commit_after>#!/usr/bin/env python
#
# Copyright 2015 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

try:
    from google.appengine.api import taskqueue, urlfetch, app_identity
    import models
except Exception as e:
    if __name__ != '__main__':
        raise e
    else:
        print "Warning: import exception '{0}'".format(e)

from urllib import quote_plus
from urlparse import urljoin, urlparse
import cgi
import datetime
import json
import logging
import lxml.etree

################################################################################

ENABLE_EXPERIMENTAL = app_identity.get_application_id().endswith('-dev')
PHYSICAL_WEB_USER_AGENT = 'Mozilla/5.0' # TODO: Find a more descriptive string.
BASE_URL = 'https://' + app_identity.get_application_id() + '.appspot.com'

################################################################################

def BuildResponse(objects):
    metadata_output = []
    unresolved_output = []

    # Resolve the devices
    for obj in objects:
        url = obj.get('url', None)
        force_update = obj.get('force', False)
        rssi = obj.get('rssi', None)
        txpower = obj.get('txpower', None)
        distance = ComputeDistance(rssi, txpower)

        def append_invalid():
            #unresolved_output.append({
            #    'id': url
            #})
            pass

        if url is None:
            continue

        parsed_url = urlparse(url)
        if parsed_url.scheme != 'http' and parsed_url.scheme != 'https':
            append_invalid()
            continue

        try:
            siteInfo = GetSiteInfoForUrl(url, distance, force_update)
        except FailedFetchException:
            append_invalid()
            continue

        if siteInfo is None:
            # It's a valid url, which we didn't fail to fetch, so it must be `No Content`
            continue

        device_data = {}
        device_data['id'] = url
        # TODO: change url to the original url (perhaps minus our goo.gl shortened values)
        device_data['url'] = siteInfo.url
        # TODO: change displayUrl to the "most applicable" url (resolve shorteners, but perhaps not all redirects)
        device_data['displayUrl'] = siteInfo.url
        if siteInfo.title is not None:
            device_data['title'] = siteInfo.title
        if siteInfo.description is not None:
            device_data['description'] = siteInfo.description
        if siteInfo.favicon_url is not None:
            device_data['icon'] = urljoin(BASE_URL, '/favicon?url=' + quote_plus(siteInfo.favicon_url))
        if siteInfo.jsonlds is not None:
            device_data['json-ld'] = json.loads(siteInfo.jsonlds)
        device_data['distance'] = distance
        device_data['groupid'] = ComputeGroupId(url, siteInfo.title, siteInfo.description)

        metadata_output.append(device_data)


    metadata_output = map(ReplaceDistanceWithRank, RankedResponse(metadata_output))

    ret = {
        "metadata": metadata_output,
    }

    if unresolved_output:
        ret["unresolved"] = unresolved_output

    return ret

################################################################################

def ComputeDistance(rssi, txpower):
    try:
        rssi = float(rssi)
        txpower = float(txpower)
        if rssi in [127, 128]: # Known invalid rssi values
            return None
        path_loss = txpower - rssi
        distance = pow(10.0, (path_loss - 41) / 20)
        return distance
    except:
        return None

def RankedResponse(metadata_output):
    def SortByDistanceCmp(a, b):
        return cmp(a['distance'], b['distance'])

    metadata_output.sort(SortByDistanceCmp)
    return metadata_output

def ReplaceDistanceWithRank(device_data):
    distance = device_data['distance']
    distance = distance if distance is not None else 1000
    device_data['rank'] = distance
    device_data.pop('distance', None)
    return device_data

################################################################################

def ComputeGroupId(url, title, description):
    import hashlib
    domain = urlparse(url).netloc
    seed = domain + title
    groupid = hashlib.sha1(seed).hexdigest()[:16]
    return groupid

################################################################################

# This is used to recursively look up in cache after each redirection.
# We don't cache the redirection itself, but we always want to cache the final destination.
def GetSiteInfoForUrl(url, distance=None, force_update=False):
    logging.info('GetSiteInfoForUrl url:{0}, distance:{1}'.format(url, distance))

    siteInfo = None

    if force_update:
        siteInfo = FetchAndStoreUrl(siteInfo, url, distance, force_update)
    else:
        siteInfo = models.SiteInformation.get_by_id(url)

        if siteInfo is None:
            siteInfo = FetchAndStoreUrl(siteInfo, url, distance, force_update)
        else:
            # If the cache is older than 5 minutes, queue a refresh
            updated_ago = datetime.datetime.now() - siteInfo.updated_on
            if updated_ago > datetime.timedelta(minutes=5):
                logging.info('Queue RefreshUrl for url: {0}, which was updated {1} ago'.format(url, updated_ago))
                # Add request to queue.
                taskqueue.add(url='/refresh-url', params={'url': url})

    return siteInfo

################################################################################
{'fraction_non_alphanumeric': 0.1601525262154433, 'fraction_numerical': 0.006153046191177745, 'mean_word_length': 3.082065794128051, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5288402', 'n_tokens_mistral': 3170, 'n_tokens_neox': 3021, 'n_words': 931}