<commit_before>#!/usr/bin/env python2
# -*- coding: utf-8 -*-

import time
from html2xml import HTML2XML, findTag, findTags
from httputil import encode_multipart_formdata
import urllib2
import socket
from json import JSONDecoder

try:
    import simplexml
except:
    from xmpp import simplexml

http_host = "http://tabun.everypony.ru"
halfclosed = ("borderline", "shipping", "erpg", "gak", "RPG", "roliplay")

headers_example = {
    "connection": "close",
    "user-agent": "Mozilla/5.0 (X11; Linux i686) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1500.52 Safari/537.36",
    
}

class NoRedirect(urllib2.HTTPRedirectHandler):
    def http_error_302(self, req, fp, code, msg, headers):
        return fp

    http_error_301 = http_error_303 = http_error_307 = http_error_302
    
global_opener = urllib2.build_opener()

class TabunError(Exception):
    def __init__(self, msg=None, code=0):
        if not msg: msg = str(code)
        Exception.__init__(self, str(msg))
        self.code = int(code)

class TabunResultError(TabunError): pass

class Post:
    def __init__(self, time, blog, post_id, author, title, body, tags, short=False, private=False, blog_name=None):
        self.time = time
        self.blog = str(blog)
        self.post_id = int(post_id)
        self.author = str(author)
        self.title = unicode(title)
        if not isinstance(body, simplexml.Node): raise ValueError
        self.body = body
        self.tags = tags
        self.short = bool(short)
        self.private = bool(private)
        self.blog_name = unicode(blog_name) if blog_name else None
        
    def __repr__(self):
        return "<post " + self.blog + "/" + str(self.post_id) + ">"
    
    def __str__(self):
        return self.__repr__()

class Comment:
    def __init__(self, time, blog, post_id, comment_id, author, body, parent_id=None, post_title=None):
        self.time = time
        self.blog = str(blog) if blog else None
        self.post_id = int(post_id)
        self.comment_id = int(comment_id)
        self.author = str(author)
        if not isinstance(body, simplexml.Node): raise ValueError
        self.body = body
        if parent_id: self.parent_id = int(parent_id)
        else: self.parent_id = None
        if post_title: self.post_title = unicode(post_title)
        else: self.post_title = None
        
    def __repr__(self):
        return "<comment " + (self.blog + "/" + str(self.post_id) + "/" if self.blog and self.post_id else "") + str(self.comment_id) + ">"

    def __str__(self):
        return self.__repr__()

class Blog:
    def __init__(self, blog_id, blog, name, creator, readers=0, rating=0.0, closed=False):
        self.blog_id = int(blog_id)
        self.blog = str(blog)
        self.name = unicode(name)
        self.creator = str(creator)
        self.readers = int(readers)
        self.rating = int(rating)
        self.closed = bool(closed)

    def __repr__(self):
        return "<blog " + self.blog + ">"

    def __str__(self):
        return self.__repr__()

class StreamItem:
    def __init__(self, blog, blog_title, title, author, comment_id, comments_count):
        self.blog = str(blog)
        self.blog_title = unicode(blog_title)
        self.title = unicode(title)
        self.author = str(author)
        self.comment_id = int(comment_id)
        self.comments_count = int(comments_count)

    def __repr__(self):
        return "<stream_item " + self.blog + "/" + str(self.comment_id) + ">"

    def __str__(self):
        return self.__repr__()

 
class User:
    phpsessid = None
    username = None
    security_ls_key = None
    
    def __init__(self, login=None, passwd=None, phpsessid=None, security_ls_key=None):
        "Допустимые комбинации параметров:"
        "- login + passwd (не реализовано)"
        "- phpsessid"
        "- login + phpsessid + security_ls_key (без запроса к серверу)"
        "- без параметров (анонимус)"
        if login or passwd:
            raise NotImplementedError("Please use PHPSESSID")
        
        self.jd = JSONDecoder()
        
        # for thread safety
        self.opener = urllib2.build_opener()
        self.noredir = urllib2.build_opener(NoRedirect)

        if (not login or not password) and not phpsessid: # anonymous
            return
       
        self.phpsessid = str(phpsessid).split(";", 1)[0]
        username = None
        
        if not security_ls_key:
            req = urllib2.Request(http_host+"/")
            for header, value in headers_example.items():
                req.add_header(header, value)
            req.add_header('cookie', "PHPSESSID=" + self.phpsessid)
            resp = self.opener.open(req, timeout=10)
            if resp.getcode() != 200: raise TabunError(code=resp.getcode())
            raw_data = resp.read()
            
            userinfo = raw_data[raw_data.find('<div class="dropdown-user"'):]
            del raw_data
            userinfo = userinfo[:userinfo.find("<nav")]
            parser = HTML2XML()
            parser.feed(userinfo)
            userinfo = parser.node.getTag("div")
            if not userinfo: raise TabunError("userinfo not found")
            
            username = userinfo.getTag("a", {"class": "username"}).getData().encode("utf-8")
            uls = userinfo.getTags("ul", {"class": "dropdown-user-menu"})
            
            security_ls_key = uls[0].getTag("li", {"class": "item-signout"}).getTag("a")['href'].\
                encode("utf-8").split("security_ls_key=", 1)[-1].split("&",1)[0]
        
        self.username = username if username else login
        self.security_ls_key = security_ls_key
    
    def check_login(self):
        if not self.phpsessid or not self.security_ls_key:
            raise TabunError("Not logined")
    
    def urlopen(self, url, data=None, headers={}, redir=True):
        if not isinstance(url, urllib2.Request):
            if url[0] == "/": url = http_host + url
            url = urllib2.Request(url, data)
        if self.phpsessid:
            url.add_header('cookie', "PHPSESSID=" + self.phpsessid)
        
        for header, value in headers_example.items(): url.add_header(header, value)
        if headers:
            for header, value in headers.items(): url.add_header(header, value)
        
        try:
            return (self.opener.open if redir else self.noredir.open)(url, timeout=10)
        except urllib2.HTTPError as exc:
            raise TabunError(code=exc.getcode())
        except urllib2.URLError as exc:
            raise TabunError(exc.reason.strerror, -exc.reason.errno if exc.reason.errno else 0)
        except socket.timeout:
            raise TabunError("Timeout", -2)
     
    def send_form(self, url, fields=(), files=(), timeout=None, headers={}, redir=True):
        if not isinstance(url, urllib2.Request):
            if url[0] == "/": url = http_host + url
            url = urllib2.Request(url)
        if self.phpsessid:
            url.add_header('cookie', "PHPSESSID=" + self.phpsessid)
        
        for header, value in headers_example.items(): url.add_header(header, value)
        if headers:
            for header, value in headers.items(): url.add_header(header, value)
            
        content_type, data = encode_multipart_formdata(fields, files)
        url.add_header('content-type', content_type)
        
        try:
            if timeout is None:
                return (self.opener.open if redir else self.noredir.open)(url, data, timeout=20)
            else:
                return (self.opener.open if redir else self.noredir.open)(url, data, timeout, timeout=20)
        except urllib2.HTTPError as exc:
            raise TabunError(code=exc.getcode())
        except urllib2.URLError as exc:
            raise TabunError(exc.reason.strerror, -exc.reason.errno if exc.reason.errno else 0)
        except socket.timeout:
            raise TabunError("Timeout", -2)
       
    def add_post(self, blog_id, title, body, tags, draft=False):
        self.check_login()
        blog_id = int(blog_id if blog_id else 0)<commit_msg>added "key" cookie<commit_after>#!/usr/bin/env python2
# -*- coding: utf-8 -*-

import time
from html2xml import HTML2XML, findTag, findTags
from httputil import encode_multipart_formdata
import urllib2
import socket
from json import JSONDecoder

try:
    import simplexml
except:
    from xmpp import simplexml

http_host = "http://tabun.everypony.ru"
halfclosed = ("borderline", "shipping", "erpg", "gak", "RPG", "roliplay")

headers_example = {
    "connection": "close",
    "user-agent": "Mozilla/5.0 (X11; Linux i686) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1500.52 Safari/537.36",
    
}

class NoRedirect(urllib2.HTTPRedirectHandler):
    def http_error_302(self, req, fp, code, msg, headers):
        return fp

    http_error_301 = http_error_303 = http_error_307 = http_error_302
    
global_opener = urllib2.build_opener()

class TabunError(Exception):
    def __init__(self, msg=None, code=0):
        if not msg: msg = str(code)
        Exception.__init__(self, str(msg))
        self.code = int(code)

class TabunResultError(TabunError): pass

class Post:
    def __init__(self, time, blog, post_id, author, title, body, tags, short=False, private=False, blog_name=None):
        self.time = time
        self.blog = str(blog)
        self.post_id = int(post_id)
        self.author = str(author)
        self.title = unicode(title)
        if not isinstance(body, simplexml.Node): raise ValueError
        self.body = body
        self.tags = tags
        self.short = bool(short)
        self.private = bool(private)
        self.blog_name = unicode(blog_name) if blog_name else None
        
    def __repr__(self):
        return "<post " + self.blog + "/" + str(self.post_id) + ">"
    
    def __str__(self):
        return self.__repr__()

class Comment:
    def __init__(self, time, blog, post_id, comment_id, author, body, parent_id=None, post_title=None):
        self.time = time
        self.blog = str(blog) if blog else None
        self.post_id = int(post_id)
        self.comment_id = int(comment_id)
        self.author = str(author)
        if not isinstance(body, simplexml.Node): raise ValueError
        self.body = body
        if parent_id: self.parent_id = int(parent_id)
        else: self.parent_id = None
        if post_title: self.post_title = unicode(post_title)
        else: self.post_title = None
        
    def __repr__(self):
        return "<comment " + (self.blog + "/" + str(self.post_id) + "/" if self.blog and self.post_id else "") + str(self.comment_id) + ">"

    def __str__(self):
        return self.__repr__()

class Blog:
    def __init__(self, blog_id, blog, name, creator, readers=0, rating=0.0, closed=False):
        self.blog_id = int(blog_id)
        self.blog = str(blog)
        self.name = unicode(name)
        self.creator = str(creator)
        self.readers = int(readers)
        self.rating = int(rating)
        self.closed = bool(closed)

    def __repr__(self):
        return "<blog " + self.blog + ">"

    def __str__(self):
        return self.__repr__()

class StreamItem:
    def __init__(self, blog, blog_title, title, author, comment_id, comments_count):
        self.blog = str(blog)
        self.blog_title = unicode(blog_title)
        self.title = unicode(title)
        self.author = str(author)
        self.comment_id = int(comment_id)
        self.comments_count = int(comments_count)

    def __repr__(self):
        return "<stream_item " + self.blog + "/" + str(self.comment_id) + ">"

    def __str__(self):
        return self.__repr__()

 
class User:
    phpsessid = None
    username = None
    security_ls_key = None
    key = None
    
    def __init__(self, login=None, passwd=None, phpsessid=None, security_ls_key=None, key=None):
        "Допустимые комбинации параметров:"
        "- login + passwd (не реализовано)"
        "- phpsessid [+ key] - без куки key разлогинивает через некоторое время"
        "- login + phpsessid + security_ls_key [+ key] (без запроса к серверу)"
        "- без параметров (анонимус)"
        if login or passwd:
            raise NotImplementedError("Please use PHPSESSID and key")
        
        self.jd = JSONDecoder()
        
        # for thread safety
        self.opener = urllib2.build_opener()
        self.noredir = urllib2.build_opener(NoRedirect)

        if (not login or not password) and not phpsessid: # anonymous
            return
       
        self.phpsessid = str(phpsessid).split(";", 1)[0]
        if key:
            self.key = str(key).split(";", 1)[0]
        username = None
        
        if not security_ls_key:
            req = urllib2.Request(http_host+"/")
            for header, value in headers_example.items():
                req.add_header(header, value)
            req.add_header('cookie', "PHPSESSID=" + self.phpsessid + ((';key='+self.key) if self.key else ''))
            resp = self.opener.open(req, timeout=10)
            if resp.getcode() != 200: raise TabunError(code=resp.getcode())
            raw_data = resp.read()
            
            userinfo = raw_data[raw_data.find('<div class="dropdown-user"'):]
            del raw_data
            userinfo = userinfo[:userinfo.find("<nav")]
            parser = HTML2XML()
            parser.feed(userinfo)
            userinfo = parser.node.getTag("div")
            if not userinfo: raise TabunError("userinfo not found")
            
            username = userinfo.getTag("a", {"class": "username"}).getData().encode("utf-8")
            uls = userinfo.getTags("ul", {"class": "dropdown-user-menu"})
            
            security_ls_key = uls[0].getTag("li", {"class": "item-signout"}).getTag("a")['href'].\
                encode("utf-8").split("security_ls_key=", 1)[-1].split("&",1)[0]
        
        self.username = username if username else login
        self.security_ls_key = security_ls_key
    
    def check_login(self):
        if not self.phpsessid or not self.security_ls_key:
            raise TabunError("Not logined")
    
    def urlopen(self, url, data=None, headers={}, redir=True):
        if not isinstance(url, urllib2.Request):
            if url[0] == "/": url = http_host + url
            url = urllib2.Request(url, data)
        if self.phpsessid:
            url.add_header('cookie', "PHPSESSID=" + self.phpsessid + ((';key='+self.key) if self.key else ''))
        
        for header, value in headers_example.items(): url.add_header(header, value)
        if headers:
            for header, value in headers.items(): url.add_header(header, value)
        
        try:
            return (self.opener.open if redir else self.noredir.open)(url, timeout=10)
        except urllib2.HTTPError as exc:
            raise TabunError(code=exc.getcode())
        except urllib2.URLError as exc:
            raise TabunError(exc.reason.strerror, -exc.reason.errno if exc.reason.errno else 0)
        except socket.timeout:
            raise TabunError("Timeout", -2)
     
    def send_form(self, url, fields=(), files=(), timeout=None, headers={}, redir=True):
        if not isinstance(url, urllib2.Request):
            if url[0] == "/": url = http_host + url
            url = urllib2.Request(url)
        if self.phpsessid:
            url.add_header('cookie', "PHPSESSID=" + self.phpsessid + ((';key='+self.key) if self.key else ''))
        
        for header, value in headers_example.items(): url.add_header(header, value)
        if headers:
            for header, value in headers.items(): url.add_header(header, value)
            
        content_type, data = encode_multipart_formdata(fields, files)
        url.add_header('content-type', content_type)
        
        try:
            if timeout is None:
                return (self.opener.open if redir else self.noredir.open)(url, data, timeout=20)
            else:
                return (self.opener.open if redir else self.noredir.open)(url, data, timeout, timeout=20)
        except urllib2.HTTPError as exc:
            raise TabunError(code=exc.getcode())
        except urllib2.URLError as exc:
            raise TabunError(exc.reason.strerror, -exc.reason.errno if exc.reason.errno else 0)
        except socket.timeout:
            raise TabunError("Timeout", -2)
       
    def add_post(self, blog_id, title, body, tags, draft=False):
        self.check_login()
        blog_id = int(blog_id if blog_id else 0)
{'fraction_non_alphanumeric': 0.1202258789240391, 'fraction_numerical': 0.011172505920213734, 'mean_word_length': 2.3916803953871497, 'pattern_counts': {'":': 14, '<': 15, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 10}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2672493', 'n_tokens_mistral': 5324, 'n_tokens_neox': 5230, 'n_words': 1212}