<commit_before>import flask
from flask import Blueprint, session, redirect, url_for, escape, request, abort, g
from flask.ext.cors import cross_origin
import urllib2
import lxml.html
from . import database
import urlparse
import json

bone_blueprint = Blueprint('bone', __name__)

@bone_blueprint.route('/link/<linkid>', methods=['GET','DELETE'])
def delete_link(linkid):
    db = database.get_db()
    linkid = int(linkid)
    result = ''
    with db.cursor() as cur:
        if request.method == 'GET':
            cur.execute('SELECT id,url,title,posted FROM links WHERE id=%s', (linkid,))
            nid,url,title,posted = cur.fetchone()
            result = dict(id=nid,url=url,title=title,posted=posted.isoformat())
        elif request.method == 'DELETE':
            result = False
            if 'username' in session:
                cur.execute('SELECT delete_link(%s,%s)', (session['username'],linkid))
                result = cur.fetchone()[0]
    db.commit()
    return json.dumps(result)
    
def clean_url(url):
    scheme, netloc, path, params, query, fragment = urlparse.urlparse(url, 'http')
    if path and not netloc:
        netloc, path = path, netloc
    return urlparse.urlunparse((scheme, netloc, path, params, query, fragment))

def get_title(url):
    data = urllib2.urlopen(url)
    etree = lxml.html.parse(data)<commit_msg>Fix Access Denied errors by changing the user-agent<commit_after>import flask
from flask import Blueprint, session, redirect, url_for, escape, request, abort, g
from flask.ext.cors import cross_origin
import urllib2
import lxml.html
from . import database
import urlparse
import json

bone_blueprint = Blueprint('bone', __name__)

@bone_blueprint.route('/link/<linkid>', methods=['GET','DELETE'])
def delete_link(linkid):
    db = database.get_db()
    linkid = int(linkid)
    result = ''
    with db.cursor() as cur:
        if request.method == 'GET':
            cur.execute('SELECT id,url,title,posted FROM links WHERE id=%s', (linkid,))
            nid,url,title,posted = cur.fetchone()
            result = dict(id=nid,url=url,title=title,posted=posted.isoformat())
        elif request.method == 'DELETE':
            result = False
            if 'username' in session:
                cur.execute('SELECT delete_link(%s,%s)', (session['username'],linkid))
                result = cur.fetchone()[0]
    db.commit()
    return json.dumps(result)
    
def clean_url(url):
    scheme, netloc, path, params, query, fragment = urlparse.urlparse(url, 'http')
    if path and not netloc:
        netloc, path = path, netloc
    return urlparse.urlunparse((scheme, netloc, path, params, query, fragment))

handler=urllib2.HTTPSHandler()
opener = urllib2.build_opener(handler)
opener.addheaders = [('User-agent', 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36')]
urllib2.install_opener(opener)

handler=urllib2.HTTPHandler()
opener = urllib2.build_opener(handler)
opener.addheaders = [('User-agent', 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36')]
urllib2.install_opener(opener)

def get_title(url):
    data = urllib2.urlopen(url)
    etree = lxml.html.parse(data)
{'fraction_non_alphanumeric': 0.13393133312712652, 'fraction_numerical': 0.01732137333745747, 'mean_word_length': 3.5872340425531917, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4468414', 'n_tokens_mistral': 1096, 'n_tokens_neox': 1052, 'n_words': 252}