<commit_before>#!/usr/bin/env python

import requests

import pony.orm as pny

from datetime import datetime as dt

database = pny.Database('sqlite', '/home/famp/git/apthunt/pdxapartments.sqlite', create_db=True)

class Apartment(database.Entity):
    ask = pny.Required(int)
    bedrooms = pny.Required(int)
    imageurls = pny.Required(str)
    title = pny.Required(str)
    lat = pny.Required(float)
    lon = pny.Required(float)
    clid = pny.Required(int, size=64)
    postdate = pny.Required(dt)
    clurl = pny.Required(str)

database.generate_mapping(create_tables=True)
pdxclurl = 'https://portland.craigslist.org/jsonsearch/apa/'

def params(minp, maxp):
    return {'min_price': minp, 'max_price': maxp}

def AptSearch(min,max):
  results = requests.get(pdxclurl,params=params(min,max))
  if results.json():
    return results.json()[0]
  else:
    return 'No json returned'

@pny.db_session
def main():
    import sys
    min = sys.argv[1]
    max = sys.argv[2]
    apartments = [i for i in AptSearch(min,max) if 'Ask' in i.keys()]
    clusters = [i for i in AptSearch(min,max) if 'Ask' not in i.keys()]
    for apartment in apartments:
      clid = apartment['PostingID']
      if not database.exists("select * from Apartment where clid = $clid"):
        apartment = Apartment(
                ask=apartment['Ask'],
                bedrooms=apartment['Bedrooms'],
                title=apartment['PostingTitle'],
                lat=apartment['Latitude'],
                lon=apartment['Longitude'],
                clid=apartment['PostingID'],
                postdate=dt.fromtimestamp(apartment['PostedDate']),
                clurl=apartment['PostingURL'],
                imageurls='testurl',
                )
    #pny.commit()

if __name__ == '__main__':
    main()
<commit_msg>un-hardcode home dir<commit_after>#!/usr/bin/env python

import requests

import pony.orm as pny

from datetime import datetime as dt

database = pny.Database('sqlite', '~/git/apthunt/pdxapartments.sqlite', create_db=True)

class Apartment(database.Entity):
    ask = pny.Required(int)
    bedrooms = pny.Required(int)
    imageurls = pny.Required(str)
    title = pny.Required(str)
    lat = pny.Required(float)
    lon = pny.Required(float)
    clid = pny.Required(int, size=64)
    postdate = pny.Required(dt)
    clurl = pny.Required(str)

database.generate_mapping(create_tables=True)
pdxclurl = 'https://portland.craigslist.org/jsonsearch/apa/'

def params(minp, maxp):
    return {'min_price': minp, 'max_price': maxp}

def AptSearch(min,max):
  results = requests.get(pdxclurl,params=params(min,max))
  if results.json():
    return results.json()[0]
  else:
    return 'No json returned'

@pny.db_session
def main():
    import sys
    min = sys.argv[1]
    max = sys.argv[2]
    apartments = [i for i in AptSearch(min,max) if 'Ask' in i.keys()]
    clusters = [i for i in AptSearch(min,max) if 'Ask' not in i.keys()]
    for apartment in apartments:
      clid = apartment['PostingID']
      if not database.exists("select * from Apartment where clid = $clid"):
        apartment = Apartment(
                ask=apartment['Ask'],
                bedrooms=apartment['Bedrooms'],
                title=apartment['PostingTitle'],
                lat=apartment['Latitude'],
                lon=apartment['Longitude'],
                clid=apartment['PostingID'],
                postdate=dt.fromtimestamp(apartment['PostedDate']),
                clurl=apartment['PostingURL'],
                imageurls='testurl',
                )
    #pny.commit()

if __name__ == '__main__':
    main()

{'fraction_non_alphanumeric': 0.13348164627363737, 'fraction_numerical': 0.0027808676307007787, 'mean_word_length': 2.892857142857143, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '172967', 'n_tokens_mistral': 1211, 'n_tokens_neox': 1187, 'n_words': 248}