<commit_before>#!/usr/bin/env python
#
# Copyright 2014 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
import webapp2
import json
import logging
from datetime import datetime, timedelta
from google.appengine.ext import ndb
from google.appengine.api import urlfetch
from urlparse import urljoin
from urlparse import urlparse
import os
import re
from lxml import etree
import cgi

class BaseModel(ndb.Model):
    added_on = ndb.DateTimeProperty(auto_now_add = True)
    updated_on = ndb.DateTimeProperty(auto_now = True)

class SiteInformation(BaseModel):
    url = ndb.StringProperty()
    favicon_url = ndb.StringProperty()
    title = ndb.StringProperty()
    description = ndb.StringProperty()

class DemoMetadata(webapp2.RequestHandler):
    def get(self):
        objects = [
            {'url': 'http://www.caltrain.com/schedules/realtime/stations/mountainviewstation-mobile.html'},
            {'url': 'http://benfry.com/distellamap/'},
            {'url': 'http://en.wikipedia.org/wiki/Le_D%C3%A9jeuner_sur_l%E2%80%99herbe'},
            {'url': 'http://sfmoma.org'}
        ]
        metadata_output = BuildResponse(objects)
        output = {
          "metadata": metadata_output
        }
        self.response.headers['Content-Type'] = 'application/json'
        json_data = json.dumps(output);
        self.response.write(json_data)

class ResolveScan(webapp2.RequestHandler):
    def post(self):
        input_data = self.request.body
        input_object = json.loads(input_data) # Data is not sanitised.
        
        if "objects" in input_object:
            objects = input_object["objects"]
        else:
            objects = []
        
        metadata_output = BuildResponse(objects)
        output = {
          "metadata": metadata_output
        }
        self.response.headers['Content-Type'] = 'application/json'
        json_data = json.dumps(output);
        self.response.write(json_data)

def BuildResponse(objects):
        metadata_output = []
        
        # Resolve the devices
        
        for obj in objects:
            key_id = None
            url = None
            force = False
            
            if "id" in obj:
                key_id = obj["id"]
            elif "url" in obj:
                key_id = obj["url"]
                url = obj["url"]
                parsed_url = urlparse(url)
                if parsed_url.scheme == '':
                    url = 'http://' + url
                    key_id = url

            if "force" in obj:
                force = True
                
            # We need to go and fetch.  We probably want to asyncly fetch.

            # We don't need RSSI yet.
            #rssi = obj["rssi"]
           
            # Really if we don't have the data we should not return it.
            siteInfo = SiteInformation.get_by_id(url)

            if force or siteInfo is None or siteInfo.updated_on < datetime.now() - timedelta(minutes=5):
                # If we don't have the data or it is older than 5 minutes, fetch.
                siteInfo = FetchAndStoreUrl(siteInfo, url)

            device_data = {};
            if siteInfo is not None:
                device_data["id"] = url
                device_data["url"] = siteInfo.url
                if siteInfo.title is not None:
                    device_data["title"] = siteInfo.title
                if siteInfo.description is not None:
                    device_data["description"] = siteInfo.description
                if siteInfo.favicon_url is not None:
                    device_data["icon"] = siteInfo.favicon_url
            else:
                device_data["id"] = url
                device_data["url"] = url

            metadata_output.append(device_data)

        return metadata_output

def FetchAndStoreUrl(siteInfo, url):
    # Index the page
    result = urlfetch.fetch(url, validate_certificate = True)<commit_msg>Bill's comments<commit_after>#!/usr/bin/env python
#
# Copyright 2014 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
import webapp2
import json
import logging
from datetime import datetime, timedelta
from google.appengine.ext import ndb
from google.appengine.api import urlfetch
from urlparse import urljoin
from urlparse import urlparse
import os
import re
from lxml import etree
import cgi
from google.appengine.api import urlfetch_errors

class BaseModel(ndb.Model):
    added_on = ndb.DateTimeProperty(auto_now_add = True)
    updated_on = ndb.DateTimeProperty(auto_now = True)

class SiteInformation(BaseModel):
    url = ndb.StringProperty()
    favicon_url = ndb.StringProperty()
    title = ndb.StringProperty()
    description = ndb.StringProperty()

class DemoMetadata(webapp2.RequestHandler):
    def get(self):
        objects = [
            {'url': 'http://www.caltrain.com/schedules/realtime/stations/mountainviewstation-mobile.html'},
            {'url': 'http://benfry.com/distellamap/'},
            {'url': 'http://en.wikipedia.org/wiki/Le_D%C3%A9jeuner_sur_l%E2%80%99herbe'},
            {'url': 'http://sfmoma.org'}
        ]
        metadata_output = BuildResponse(objects)
        output = {
          "metadata": metadata_output
        }
        self.response.headers['Content-Type'] = 'application/json'
        json_data = json.dumps(output);
        self.response.write(json_data)

class ResolveScan(webapp2.RequestHandler):
    def post(self):
        input_data = self.request.body
        input_object = json.loads(input_data) # Data is not sanitised.
        
        if "objects" in input_object:
            objects = input_object["objects"]
        else:
            objects = []
        
        metadata_output = BuildResponse(objects)
        output = {
          "metadata": metadata_output
        }
        self.response.headers['Content-Type'] = 'application/json'
        json_data = json.dumps(output);
        self.response.write(json_data)

def BuildResponse(objects):
        metadata_output = []
        
        # Resolve the devices
        
        for obj in objects:
            key_id = None
            url = None
            force = False
            valid = True
            siteInfo = None

            if "id" in obj:
                key_id = obj["id"]
            elif "url" in obj:
                key_id = obj["url"]
                url = obj["url"]
                parsed_url = urlparse(url)
                if parsed_url.scheme != 'http' and parsed_url.scheme != 'https':
                    valid = False

            if "force" in obj:
                force = True
                
            # We need to go and fetch.  We probably want to asyncly fetch.

            # We don't need RSSI yet.
            #rssi = obj["rssi"]

            if valid:
                # Really if we don't have the data we should not return it.
                siteInfo = SiteInformation.get_by_id(url)

                if force or siteInfo is None or siteInfo.updated_on < datetime.now() - timedelta(minutes=5):
                    # If we don't have the data or it is older than 5 minutes, fetch.
                    siteInfo = FetchAndStoreUrl(siteInfo, url)

            device_data = {};
            if siteInfo is not None:
                device_data["id"] = url
                device_data["url"] = siteInfo.url
                if siteInfo.title is not None:
                    device_data["title"] = siteInfo.title
                if siteInfo.description is not None:
                    device_data["description"] = siteInfo.description
                if siteInfo.favicon_url is not None:
                    device_data["icon"] = siteInfo.favicon_url
            else:
                device_data["id"] = url
                device_data["url"] = url

            metadata_output.append(device_data)

        return metadata_output

def FetchAndStoreUrl(siteInfo, url):
    # Index the page
    try:
        result = urlfetch.fetch(url, validate_certificate = True)
    except urlfetch_errors.DeadlineExceededError:
        return None

{'fraction_non_alphanumeric': 0.08973932334997227, 'fraction_numerical': 0.004437049362174155, 'mean_word_length': 2.045945945945946, 'pattern_counts': {'":': 4, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1959178', 'n_tokens_mistral': 2520, 'n_tokens_neox': 2426, 'n_words': 708}