<commit_before>#!/usr/bin/python -u
# -*- coding: utf-8 -*-

import cairo
import cgi
import mapnik
import os
import shutil
import sys
import tempfile
import resource
import signal

# Limit maximum CPU time
# The Postscript output format can sometimes take hours
resource.setrlimit(resource.RLIMIT_CPU,(180,180))

# Limit memory usage
# Some odd requests can cause extreme memory usage
resource.setrlimit(resource.RLIMIT_AS,(4000000000, 4000000000))

# Routine to output HTTP headers
def output_headers(content_type, filename = "", length = 0):
  print "Content-Type: %s" % content_type
  if filename:
    print "Content-Disposition: attachment; filename=\"%s\"" % filename
  if length:
    print "Content-Length: %d" % length
  print ""

# Routine to output the contents of a file
def output_file(file):
  file.seek(0)
  shutil.copyfileobj(file, sys.stdout)

# Routine to get the size of a file
def file_size(file):
  return os.fstat(file.fileno()).st_size

# Routine to report an error
def output_error(message, status = "400 Bad Request"):
  print "Status: %s" % status
  output_headers("text/html")
  print "<html>"
  print "<head>"
  print "<title>Error</title>"
  print "</head>"
  print "<body>"
  print "<h1>Error</h1>"
  print "<p>%s</p>" % message
  print "</body>"
  print "</html>"

# Parse CGI parameters
form = cgi.FieldStorage()

# Make sure we have a user agent
if not os.environ.has_key('HTTP_USER_AGENT'):
  os.environ['HTTP_USER_AGENT'] = 'NONE'

# Make sure we have a referer
if not os.environ.has_key('HTTP_REFERER'):
  os.environ['HTTP_REFERER'] = 'NONE'

# Get the load average
cputimes = [float(n) for n in open("/proc/stat").readline().rstrip().split()[1:-1]]
idletime = cputimes[3] / sum(cputimes)

# Process the request
if idletime < 0.2:
  # Abort if the CPU idle time on the machine is too low
  output_error("The server is too busy at the moment. Please wait a few minutes before trying again.", "503 Service Unavailable")
elif form.getvalue("scale") == "3600":
  # Block scraper
  output_error("The server is too busy at the moment. Please wait a few minutes before trying again.", "503 Service Unavailable")
<% @blocks["user_agents"].each do |user_agent| -%>
elif os.environ['HTTP_USER_AGENT'] == '<%= user_agent %>':
  # Block scraper
  output_error("The server is too busy at the moment. Please wait a few minutes before trying again.", "503 Service Unavailable")
<% end -%>
<% @blocks["referers"].each do |referer| -%>
elif os.environ['HTTP_REFERER'] == '<%= referer %>':
  # Block scraper
  output_error("The server is too busy at the moment. Please wait a few minutes before trying again.", "503 Service Unavailable")
<% end -%>
elif not form.has_key("bbox"):
  # No bounding box specified
  output_error("No bounding box specified")
elif not form.has_key("scale"):
  # No scale specified
  output_error("No scale specified")
elif not form.has_key("format"):<commit_msg>Remove block on scale 3600<commit_after>#!/usr/bin/python -u
# -*- coding: utf-8 -*-

import cairo
import cgi
import mapnik
import os
import shutil
import sys
import tempfile
import resource
import signal

# Limit maximum CPU time
# The Postscript output format can sometimes take hours
resource.setrlimit(resource.RLIMIT_CPU,(180,180))

# Limit memory usage
# Some odd requests can cause extreme memory usage
resource.setrlimit(resource.RLIMIT_AS,(4000000000, 4000000000))

# Routine to output HTTP headers
def output_headers(content_type, filename = "", length = 0):
  print "Content-Type: %s" % content_type
  if filename:
    print "Content-Disposition: attachment; filename=\"%s\"" % filename
  if length:
    print "Content-Length: %d" % length
  print ""

# Routine to output the contents of a file
def output_file(file):
  file.seek(0)
  shutil.copyfileobj(file, sys.stdout)

# Routine to get the size of a file
def file_size(file):
  return os.fstat(file.fileno()).st_size

# Routine to report an error
def output_error(message, status = "400 Bad Request"):
  print "Status: %s" % status
  output_headers("text/html")
  print "<html>"
  print "<head>"
  print "<title>Error</title>"
  print "</head>"
  print "<body>"
  print "<h1>Error</h1>"
  print "<p>%s</p>" % message
  print "</body>"
  print "</html>"

# Parse CGI parameters
form = cgi.FieldStorage()

# Make sure we have a user agent
if not os.environ.has_key('HTTP_USER_AGENT'):
  os.environ['HTTP_USER_AGENT'] = 'NONE'

# Make sure we have a referer
if not os.environ.has_key('HTTP_REFERER'):
  os.environ['HTTP_REFERER'] = 'NONE'

# Get the load average
cputimes = [float(n) for n in open("/proc/stat").readline().rstrip().split()[1:-1]]
idletime = cputimes[3] / sum(cputimes)

# Process the request
if idletime < 0.2:
  # Abort if the CPU idle time on the machine is too low
  output_error("The server is too busy at the moment. Please wait a few minutes before trying again.", "503 Service Unavailable")
<% @blocks["user_agents"].each do |user_agent| -%>
elif os.environ['HTTP_USER_AGENT'] == '<%= user_agent %>':
  # Block scraper
  output_error("The server is too busy at the moment. Please wait a few minutes before trying again.", "503 Service Unavailable")
<% end -%>
<% @blocks["referers"].each do |referer| -%>
elif os.environ['HTTP_REFERER'] == '<%= referer %>':
  # Block scraper
  output_error("The server is too busy at the moment. Please wait a few minutes before trying again.", "503 Service Unavailable")
<% end -%>
elif not form.has_key("bbox"):
  # No bounding box specified
  output_error("No bounding box specified")
elif not form.has_key("scale"):
  # No scale specified
  output_error("No scale specified")
elif not form.has_key("format"):
{'fraction_non_alphanumeric': 0.13231642426392337, 'fraction_numerical': 0.01897836112096488, 'mean_word_length': 4.572134387351778, 'pattern_counts': {'":': 1, '<': 41, '<?xml version=': 0, '>': 39, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 4, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3214754', 'n_tokens_mistral': 1877, 'n_tokens_neox': 1813, 'n_words': 613}