<commit_before>#!/usr/bin/python3
# -*- coding: utf-8 -*-

#   Copyright 2012 codestation
#
#   Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.

import re
import json
import hashlib
import logging
from urllib.request import urlopen
from urllib.error import URLError, HTTPError
from time import sleep, time, gmtime, strftime

class Api(object):

    post_api = '/post/index.json'
    tag_api = '/tag/index.json'

    _delta_time = 0
    WAIT_TIME = 1.2

    def __init__(self, host, username, password, salt):
        self.host = host
        self.username = username
        self.password = password
        self.salt = salt
        
    def _wait(self):
        self._delta_time = time() - self._delta_time
        if self._delta_time < self.WAIT_TIME:
            sleep(self.WAIT_TIME - self._delta_time)

    def _loginData(self):
        sha1data = hashlib.sha1((self.salt % self.password).encode('utf8'))
        sha1_password = <PASSWORD>.<PASSWORD>()
        return '&login=%s&password_hash=%s' % (self.username, sha1_password)
    
    def _getPosts(self, tags, page, limit):
        results = open('post.json', 'rb').read().decode('utf8')
        posts = json.loads(results)
        for post in posts:
            post['tags'] = post['tags'].split(' ')
            if not "has_comments" in post:
                post['has_comments'] = None
            if not "has_notes" in post:
                post['has_notes'] = None
        return posts
    
    def getPostsPage(self, tags, page, limit, blacklist = None, whitelist = None):
        tags = ','.join(tags)
        url = self.host + self.post_api + '?tags=%s&page=%i&limit=%i' % (tags, page, limit) + self._loginData()
        return self.getPosts(url, blacklist, whitelist)
        
    def getPostsBefore(self, post_id, tags, limit, blacklist = None, whitelist = None):
        tags = ','.join(tags)     
        url = self.host + self.post_api + '?before_id=%i&tags=%s&limit=%i' % (post_id, tags, limit) + self._loginData()
        return self.getPosts(url, blacklist, whitelist)
    
    def getTagsBefore(self, post_id, tags, limit):
        pass  
        #tags = ','.join(tags)      
        #url = self.host + self.tag_api + '?before_id=%i&tags=%s&limit=%i' % (post_id, tags, limit) + self._loginData()
        #return self.getPosts(url)
        
    def getPosts(self, url, blacklist, whitelist):<commit_msg>More API cleanup<commit_after>#!/usr/bin/python3
# -*- coding: utf-8 -*-

#   Copyright 2012 codestation
#
#   Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.

import re
import json
import hashlib
import logging
from urllib.request import urlopen
from urllib.error import URLError, HTTPError
from time import sleep, time, gmtime, strftime

class Api(object):

    post_api = '/post/index.json'
    tag_api = '/tag/index.json'

    _delta_time = 0
    WAIT_TIME = 1.2

    def __init__(self, host, username, password, salt):
        self.host = host
        self.username = username
        self.password = password
        self.salt = salt
        
    def _wait(self):
        self._delta_time = time() - self._delta_time
        if self._delta_time < self.WAIT_TIME:
            sleep(self.WAIT_TIME - self._delta_time)

    def _loginData(self):
        sha1data = hashlib.sha1((self.salt % self.password).encode('utf8'))
        sha1_password = <PASSWORD>()
        return '&login=%s&password_hash=%s' % (self.username, sha1_password)
    
    def getPostsPage(self, tags, page, limit, blacklist = None, whitelist = None):
        tags = ','.join(tags)
        url = self.host + self.post_api + '?tags=%s&page=%i&limit=%i' % (tags, page, limit) + self._loginData()
        return self.getPosts(url, blacklist, whitelist)
        
    def getPostsBefore(self, post_id, tags, limit, blacklist = None, whitelist = None):
        tags = ','.join(tags)     
        url = self.host + self.post_api + '?before_id=%i&tags=%s&limit=%i' % (post_id, tags, limit) + self._loginData()
        return self.getPosts(url, blacklist, whitelist)
    
    def getTagsBefore(self, post_id, tags, limit):
        pass  
        
    def getPosts(self, url, blacklist, whitelist):
{'fraction_non_alphanumeric': 0.11681518791166215, 'fraction_numerical': 0.007167764432390546, 'mean_word_length': 2.8159645232815964, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '754986', 'n_tokens_mistral': 1628, 'n_tokens_neox': 1578, 'n_words': 465}