<commit_before># -*- coding: utf-8 -*-
import datetime
from typing import Optional
from owlmixin import OwlMixin, TOption
from owlmixin.owlcollections import TList, TDict
from owlmixin.owlenum import OwlEnum
from requests.structures import CaseInsensitiveDict as RequestsCaseInsensitiveDict


class CaseInsensitiveDict(RequestsCaseInsensitiveDict):
    pass


class Status(OwlEnum):
    SAME = "same"
    DIFFERENT = "different"
    FAILURE = "failure"


class AccessPoint(OwlMixin):
    name: str
    host: str
    proxy: TOption[str]


class OutputSummary(OwlMixin):
    response_dir: str
    encoding: str = 'utf8'
    logger: TOption[any]


class Concurrency(OwlMixin):
    threads: int
    processes: int


class Addon(OwlMixin):
    name: str
    cls_name: str = 'Executor'
    config: TOption[dict]
    include: TOption[str]


# List is None...
class Addons(OwlMixin):
    log2reqs: Addon
    reqs2reqs: TList[Addon] = []
    res2res: TList[Addon] = []
    res2dict: TList[Addon] = []
    judgement: TList[Addon] = []
    store_criterion: TList[Addon] = []
    dump: TList[Addon] = []
    did_challenge: TList[Addon] = []
    final: TList[Addon] = []


class Config(OwlMixin):
    one: AccessPoint
    other: AccessPoint
    output: OutputSummary
    threads: int = 1
    processes: TOption[int]
    title: TOption[str]
    description: TOption[str]
    tags: TOption[TList[str]]
    input_files: TOption[TList[str]]
    addons: Addons


# --------


class Args(OwlMixin):
    run: bool
    files: TOption[TList[str]]
    title: TOption[str]
    description: TOption[str]
    config: TOption[TList[str]]
    tag: TOption[TList[str]]
    threads: TOption[int]
    processes: TOption[int]
    retry: bool
    report: TOption[str]  # Only case in which retry is True
    init: bool
    name: TOption[str]  # Only case in which init is True

    @classmethod
    def ___threads(cls, v: Optional[str]) -> int:
        return int(v) if v else None

    @classmethod
    def ___processes(cls, v: Optional[str]) -> int:
        return int(v) if v else None


# or {}
class Request(OwlMixin):
    name: TOption[str]
    path: str
    qs: TDict[TList[str]] = {}
    headers: TDict[str] = {}


class Proxy(OwlMixin):
    http: str = None
    https: str = None

    @classmethod
    def from_host(cls, host: TOption[str]) -> 'Proxy':
        return Proxy.from_dict({
            'http': f"http://{host.get()}",
            'https': f"https://{host.get()}",
        }) if not host.is_none() else None


class Response(OwlMixin):
    body: bytes
    encoding: TOption[str]
    text: str
    headers: CaseInsensitiveDict
    url: str
    status_code: int
    elapsed: datetime.timedelta

    @property
    def content_type(self) -> TOption[str]:
        return TOption(self.headers.get('content-type'))

    @property
    def mime_type(self) -> TOption[str]:
        return self.content_type.map(lambda x: x.split(';')[0])

    @classmethod
    def ___headers(cls, v):
        return CaseInsensitiveDict(v)

    @classmethod
    def from_requests(cls, res: any) -> 'Response':
        return Response.from_dict({
            'body': res.content,
            'encoding': res.encoding,
            'text': res.text,
            'headers': res.headers,
            'url': res.url,
            'status_code': res.status_code,
            'elapsed': res.elapsed,
        })


# --------

class ChallengeArg(OwlMixin):
    seq: int
    number_of_request: int
    key: str
    session: object
    req: Request
    host_one: str
    host_other: str
    proxy_one: TOption[Proxy]
    proxy_other: TOption[Proxy]
    res_dir: str

# --------<commit_msg>:arrow_upper_right: Fix performance issue (requests.text is slow)<commit_after># -*- coding: utf-8 -*-
import datetime
from typing import Optional
from owlmixin import OwlMixin, TOption
from owlmixin.owlcollections import TList, TDict
from owlmixin.owlenum import OwlEnum
from requests.structures import CaseInsensitiveDict as RequestsCaseInsensitiveDict


class CaseInsensitiveDict(RequestsCaseInsensitiveDict):
    pass


class Status(OwlEnum):
    SAME = "same"
    DIFFERENT = "different"
    FAILURE = "failure"


class AccessPoint(OwlMixin):
    name: str
    host: str
    proxy: TOption[str]


class OutputSummary(OwlMixin):
    response_dir: str
    encoding: str = 'utf8'
    logger: TOption[any]


class Concurrency(OwlMixin):
    threads: int
    processes: int


class Addon(OwlMixin):
    name: str
    cls_name: str = 'Executor'
    config: TOption[dict]
    include: TOption[str]


# List is None...
class Addons(OwlMixin):
    log2reqs: Addon
    reqs2reqs: TList[Addon] = []
    res2res: TList[Addon] = []
    res2dict: TList[Addon] = []
    judgement: TList[Addon] = []
    store_criterion: TList[Addon] = []
    dump: TList[Addon] = []
    did_challenge: TList[Addon] = []
    final: TList[Addon] = []


class Config(OwlMixin):
    one: AccessPoint
    other: AccessPoint
    output: OutputSummary
    threads: int = 1
    processes: TOption[int]
    title: TOption[str]
    description: TOption[str]
    tags: TOption[TList[str]]
    input_files: TOption[TList[str]]
    addons: Addons


# --------


class Args(OwlMixin):
    run: bool
    files: TOption[TList[str]]
    title: TOption[str]
    description: TOption[str]
    config: TOption[TList[str]]
    tag: TOption[TList[str]]
    threads: TOption[int]
    processes: TOption[int]
    retry: bool
    report: TOption[str]  # Only case in which retry is True
    init: bool
    name: TOption[str]  # Only case in which init is True

    @classmethod
    def ___threads(cls, v: Optional[str]) -> int:
        return int(v) if v else None

    @classmethod
    def ___processes(cls, v: Optional[str]) -> int:
        return int(v) if v else None


# or {}
class Request(OwlMixin):
    name: TOption[str]
    path: str
    qs: TDict[TList[str]] = {}
    headers: TDict[str] = {}


class Proxy(OwlMixin):
    http: str = None
    https: str = None

    @classmethod
    def from_host(cls, host: TOption[str]) -> 'Proxy':
        return Proxy.from_dict({
            'http': f"http://{host.get()}",
            'https': f"https://{host.get()}",
        }) if not host.is_none() else None


class Response(OwlMixin):
    body: bytes
    encoding: TOption[str]
    text: str
    headers: CaseInsensitiveDict
    url: str
    status_code: int
    elapsed: datetime.timedelta

    @property
    def content_type(self) -> TOption[str]:
        return TOption(self.headers.get('content-type'))

    @property
    def mime_type(self) -> TOption[str]:
        return self.content_type.map(lambda x: x.split(';')[0])

    @classmethod
    def ___headers(cls, v):
        return CaseInsensitiveDict(v)

    @classmethod
    def from_requests(cls, res: any) -> 'Response':
        return Response.from_dict({
            'body': res.content,
            'encoding': res.encoding,
            'text': res.content.decode(res.encoding or 'utf8'),
            'headers': res.headers,
            'url': res.url,
            'status_code': res.status_code,
            'elapsed': res.elapsed,
        })


# --------

class ChallengeArg(OwlMixin):
    seq: int
    number_of_request: int
    key: str
    session: object
    req: Request
    host_one: str
    host_other: str
    proxy_one: TOption[Proxy]
    proxy_other: TOption[Proxy]
    res_dir: str

# --------
{'fraction_non_alphanumeric': 0.12384448069603045, 'fraction_numerical': 0.002311038607939097, 'mean_word_length': 2.7805755395683454, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 15, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3779873', 'n_tokens_mistral': 2522, 'n_tokens_neox': 2470, 'n_words': 608}