<commit_before>import re
import requests
from pygments import highlight
from pygments.lexers import guess_lexer, guess_lexer_for_filename
from pygments.formatters import HtmlFormatter
import boto3

s3 = boto3.resource('s3')

def validate_link(link):
  pattern = re.compile("^https:\/\/github\.com\/([a-zA-Z0-9.\-_]*)\/([a-zA-Z0-9.\-_]*)\/blob\/([a-zA-Z0-9.\-_]*)([a-zA-Z0-9.\/\-_]*)\/([a-zA-Z0-9.\-_]*)#L(\d*)-L(\d*)$")
  match = pattern.match(link)
  if match:
    return (match.group(1), match.group(2), match.group(3), match.group(5), int(match.group(6)), int(match.group(7)))
  else:
    raise Exception("Link provided was not a valid github link.")

def handler(event, context):
  github_link = event['github_link']

  owner, repo, branch, filename, start_line, end_line = validate_link(github_link)
  request = requests.get("https://api.github.com/repos/%s/%s/contents/%s?ref=%s" % (owner, repo, filename, branch)).json()
  file = request['content'].decode(request['encoding'])
  chunk = '\n'.join(file.splitlines()[start_line-1:end_line])

  lexer = guess_lexer_for_filename(filename, chunk)
  formatter = HtmlFormatter(linenos=True)
  html = highlight(chunk, lexer, formatter)<commit_msg>fix path regex<commit_after>import re
import requests
from pygments import highlight
from pygments.lexers import guess_lexer, guess_lexer_for_filename
from pygments.formatters import HtmlFormatter
import boto3

s3 = boto3.resource('s3')

def validate_link(link):
  pattern = re.compile("^https:\/\/github\.com\/([a-zA-Z0-9.\-_]*)\/([a-zA-Z0-9.\-_]*)\/blob\/([a-zA-Z0-9.\-_]*)([a-zA-Z0-9.\/\-_]*)\/([a-zA-Z0-9.\-_]*)#L(\d*)-L(\d*)$")
  match = pattern.match(link)
  if match:
    return (match.group(1), match.group(2), match.group(3), match.group(4), match.group(5), int(match.group(6)), int(match.group(7)))
  else:
    raise Exception("Link provided was not a valid github link.")

def handler(event, context):
  github_link = event['github_link']

  owner, repo, branch, path, filename, start_line, end_line = validate_link(github_link)
  request = requests.get("https://api.github.com/repos/%s/%s/contents%s/%s?ref=%s" % (owner, repo, path, filename, branch)).json()
  print "https://api.github.com/repos/%s/%s/contents/%s?ref=%s" % (owner, repo, filename, branch)
  file = request['content'].decode(request['encoding'])
  chunk = '\n'.join(file.splitlines()[start_line-1:end_line])

  lexer = guess_lexer_for_filename(filename, chunk)
  formatter = HtmlFormatter(linenos=True)
  html = highlight(chunk, lexer, formatter)
{'fraction_non_alphanumeric': 0.19793732645775486, 'fraction_numerical': 0.01705672352241174, 'mean_word_length': 4.824480369515012, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 3, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4759138', 'n_tokens_mistral': 954, 'n_tokens_neox': 919, 'n_words': 164}