<commit_before>cheerio = require 'cheerio'
request = require 'request'
express = require 'express'
NodeCache = require 'node-cache'
fs = require 'fs'

app = express()
cache = new NodeCache()

tierheim_url = "http://www.tierfreunde-helfen.de/index.php?zuhausegesucht-tiere-in-not"
splitpos = tierheim_url.lastIndexOf '/'
base_url = tierheim_url.slice 0, splitpos+1

# Get details for one pet
get_details = (url)->
    new Promise (f, r) ->
        request url, (err, response, body) ->
            if err
                r err
            $ = cheerio.load body
            content = $('#content')
            name = content.find('h1').text()
            img = content.find('img').attr('src')
            pic = base_url + content.find('img').attr('src')
            id = img.split '.', 1
            id = id[0].split '/'
            content.find('h1').remove()
            content.find('a').remove()
            details =
                id: id[-1..][0]
                pic: pic
                name: name
                link: url
                desc: content
                    .text().replace(/\n/g, '')
                    .replace(/\r/g, '')
                    .replace(/\t/g, '')
                    .trim()
            f details

# Get all urls for pets
get_detailUrls = (url) ->
    new Promise (f, r) ->
        request url, (err, response, body) ->
            if err
                r err
            urls = []
            $ = cheerio.load body
            $('.teaser-subline').each ->
                elem = $(this)
                detail_url = base_url + elem.find('.teaser-image').find('a').attr('href')
                urls.push detail_url
            f urls

# Get the data from cache or save data in cache if data is none
# cache lifetime: 24h
get_data = ->
    new Promise (f, r) ->
        values = cache.get('tiere')
        if not values
            get_detailUrls tierheim_url
                .then (urls) ->
                    p = []
                    for url in urls
                        p.push get_details url
                    Promise.all p
                .then (values) ->
                    cache.set('tiere', values, 60*60*24)
                    f values
                .catch err ->
                    r err
        else
            f values

# get a pet that was not posted yet
get_notPostedPet = ->
    new Promise (f, r) ->
        filename = 'posted_pets.json'
        try
            if fs.existsSync(filename)
                postedPets = JSON.parse fs.readFileSync filename, 'utf8'
            else
                fs.writeFileSync filename, JSON.stringify []
                postedPets = []
        catch err
            r err
        get_data().then (pets) ->
            notPostedPets = []
            for pet in pets<commit_msg>fixed catch<commit_after>cheerio = require 'cheerio'
request = require 'request'
express = require 'express'
NodeCache = require 'node-cache'
fs = require 'fs'

app = express()
cache = new NodeCache()

tierheim_url = "http://www.tierfreunde-helfen.de/index.php?zuhausegesucht-tiere-in-not"
splitpos = tierheim_url.lastIndexOf '/'
base_url = tierheim_url.slice 0, splitpos+1

# Get details for one pet
get_details = (url)->
    new Promise (f, r) ->
        request url, (err, response, body) ->
            if err
                r err
            $ = cheerio.load body
            content = $('#content')
            name = content.find('h1').text()
            img = content.find('img').attr('src')
            pic = base_url + content.find('img').attr('src')
            id = img.split '.', 1
            id = id[0].split '/'
            content.find('h1').remove()
            content.find('a').remove()
            details =
                id: id[-1..][0]
                pic: pic
                name: name
                link: url
                desc: content
                    .text().replace(/\n/g, '')
                    .replace(/\r/g, '')
                    .replace(/\t/g, '')
                    .trim()
            f details

# Get all urls for pets
get_detailUrls = (url) ->
    new Promise (f, r) ->
        request url, (err, response, body) ->
            if err
                r err
            urls = []
            $ = cheerio.load body
            $('.teaser-subline').each ->
                elem = $(this)
                detail_url = base_url + elem.find('.teaser-image').find('a').attr('href')
                urls.push detail_url
            f urls

# Get the data from cache or save data in cache if data is none
# cache lifetime: 24h
get_data = ->
    new Promise (f, r) ->
        values = cache.get('tiere')
        if not values
            get_detailUrls tierheim_url
                .then (urls) ->
                    p = []
                    for url in urls
                        p.push get_details url
                    Promise.all p
                .then (values) ->
                    cache.set('tiere', values, 60*60*24)
                    f values
                .catch (err) ->
                    r err
        else
            f values

# get a pet that was not posted yet
get_notPostedPet = ->
    new Promise (f, r) ->
        filename = 'posted_pets.json'
        try
            if fs.existsSync(filename)
                postedPets = JSON.parse fs.readFileSync filename, 'utf8'
            else
                fs.writeFileSync filename, JSON.stringify []
                postedPets = []
        catch err
            r err
        get_data().then (pets) ->
            notPostedPets = []
            for pet in pets
{'fraction_non_alphanumeric': 0.117499101688825, 'fraction_numerical': 0.006108515989938915, 'mean_word_length': 1.2759607522485692, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 33, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7448806', 'n_tokens_mistral': 1614, 'n_tokens_neox': 1535, 'n_words': 420}