<commit_before>#!/usr/bin/python

#from HTMLParser import HTMLParser
import requests
from lxml import html
import jinja2
import boto

URL = 'https://launchpad.net/~rquillo/+archive/ansible/+packages'

data = {}
packages = requests.get(URL)
tree = html.fromstring(packages.text)

data['author']= tree.xpath("//*[@itemprop='breadcrumb']/li[1]/a/text()")[0]
data['title'] = ' : '.join([x.strip() for x in tree.xpath("//*[@itemprop='breadcrumb']/li//text()") if x.strip()])

data['items'] = []
for row in tree.xpath("//*[@id='packages_list']/*//tr[contains(@class,'archive_package_row')]"):
  rowstrs = [x.strip() for x in row.xpath("td//text()") if x.strip()]
  if not rowstrs:
    continue
  data['items'].append( (
    rowstrs[0][0],
    URL,
    ' '.join( (rowstrs) )
  ))


# idea stolen from codeape on stackoverflow: http://stackoverflow.com/a/2101186/659298
curr_dir = os.path.dirname(os.path.realpath(__file__))
output_atom = jinja2.Environment(loader=jinja2.FileSystemLoader(curr_dir)).get_template("atomtemplate.xml.j2").render(data)

s3 = boto.connect_s3()
s3key = s3.get_bucket('tedder').new_key('rss/ppa/rquillo.atom')
s3key.set_metadata('Content-Type', 'application/atom+xml')<commit_msg>get os<commit_after>#!/usr/bin/python

#from HTMLParser import HTMLParser
import requests
from lxml import html
import jinja2
import boto
import os

URL = 'https://launchpad.net/~rquillo/+archive/ansible/+packages'

data = {}
packages = requests.get(URL)
tree = html.fromstring(packages.text)

data['author']= tree.xpath("//*[@itemprop='breadcrumb']/li[1]/a/text()")[0]
data['title'] = ' : '.join([x.strip() for x in tree.xpath("//*[@itemprop='breadcrumb']/li//text()") if x.strip()])

data['items'] = []
for row in tree.xpath("//*[@id='packages_list']/*//tr[contains(@class,'archive_package_row')]"):
  rowstrs = [x.strip() for x in row.xpath("td//text()") if x.strip()]
  if not rowstrs:
    continue
  data['items'].append( (
    rowstrs[0][0],
    URL,
    ' '.join( (rowstrs) )
  ))


# idea stolen from codeape on stackoverflow: http://stackoverflow.com/a/2101186/659298
curr_dir = os.path.dirname(os.path.realpath(__file__))
output_atom = jinja2.Environment(loader=jinja2.FileSystemLoader(curr_dir)).get_template("atomtemplate.xml.j2").render(data)

s3 = boto.connect_s3()
s3key = s3.get_bucket('tedder').new_key('rss/ppa/rquillo.atom')
s3key.set_metadata('Content-Type', 'application/atom+xml')
{'fraction_non_alphanumeric': 0.2007496876301541, 'fraction_numerical': 0.021657642648896292, 'mean_word_length': 5.321052631578947, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 6}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4278779', 'n_tokens_mistral': 925, 'n_tokens_neox': 888, 'n_words': 144}