<commit_before>import os
from flask import Flask, Response
import requests

app = Flask(__name__)

@app.route("/<path:doc_link>",methods=["GET"])
def get_doc(doc_link):
    #the doc_link is the unique part of the pdf's url.
    #so for example, for the document at:
    #https://api.iga.in.gov/2015/bills/hb1001/versions
        #/hb1001.02.comh?format=pdf

    #the url here will be:
    # in.proxy.openstates.org/2015/bills/hb1001/versions/hb1001.02.comh

    #also note that as of right now, their site fails https verification

    headers = {}
    headers['Authorization'] = os.environ['INDIANA_API_KEY']
    headers['Content-Type'] = "application/pdf"
    full_link = "https://api.iga.in.gov/" + doc_link + "?format=pdf"
    page = requests.get(full_link,headers=headers,verify=False)
    
    if page.status_code == 429:
        resp = Response()
        resp.status_code = 429
        resp.headers = page.headers
        return resp

    if page.status_code != 200:
        resp = Response()
        resp.status_code = 500
        return resp

    resp = Response(page.content)
    resp.headers["Content-Type"] = "application/pdf"
    return resp

@app.route("/")
def index():
    description = """Accessing Indiana's legislative documents \
                    without an API key is hard to do in a consistent \
                    way. This service requests the desired document \
                    using Open States' API key and make them searchable \
                    in the Open States interface."""

    return(description)


if __name__ == "__main__":
    app.run(debug=False)<commit_msg>don't allow robots to crawl this proxy<commit_after>import os
from flask import Flask, Response
import requests

app = Flask(__name__)

@app.route("/<path:doc_link>",methods=["GET"])
def get_doc(doc_link):
    #the doc_link is the unique part of the pdf's url.
    #so for example, for the document at:
    #https://api.iga.in.gov/2015/bills/hb1001/versions
        #/hb1001.02.comh?format=pdf

    #the url here will be:
    # in.proxy.openstates.org/2015/bills/hb1001/versions/hb1001.02.comh

    #also note that as of right now, their site fails https verification

    headers = {}
    headers['Authorization'] = os.environ['INDIANA_API_KEY']
    headers['Content-Type'] = "application/pdf"
    full_link = "https://api.iga.in.gov/" + doc_link + "?format=pdf"
    page = requests.get(full_link,headers=headers,verify=False)
    
    if page.status_code == 429:
        resp = Response()
        resp.status_code = 429
        resp.headers = page.headers
        return resp

    if page.status_code != 200:
        resp = Response()
        resp.status_code = 500
        return resp

    resp = Response(page.content)
    resp.headers["Content-Type"] = "application/pdf"
    resp.headers["X-Robots-Tag"] = "noindex"
    return resp

@app.route("/")
def index():
    description = """Accessing Indiana's legislative documents \
                    without an API key is hard to do in a consistent \
                    way. This service requests the desired document \
                    using Open States' API key and make them searchable \
                    in the Open States interface."""

    return(description)

@app.route("/robots.txt")
def robots.txt():
    return """User-agent: *
Disallow: /"""



if __name__ == "__main__":
    app.run(debug=False)

{'fraction_non_alphanumeric': 0.11729857819905214, 'fraction_numerical': 0.023696682464454975, 'mean_word_length': 2.6233905579399144, 'pattern_counts': {'":': 2, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2150974', 'n_tokens_mistral': 1097, 'n_tokens_neox': 992, 'n_words': 279}