<commit_before>import requests
import uuid
import io
import types
import os.path

from . import constants, exceptions, models
from .lib import mxliff


__all__ = ('Auth', 'Client', 'Domain', 'Project', 'Job', 'TranslationMemory', 'Asynchronous',
           'Analysis')


class BaseApi(object):
    """Inheriting classes must have the api_version attribute"""
    def __init__(self, token: {'Authentication token for using APIs': str}):
        if not hasattr(self, 'api_version'):
            # This exception is for development this library.
            raise NotImplementedError(
                'api_version is not set in {}'.format(self.__class__.__name__))

        self.token = token

    def _make_url(self, *args, **kwargs):
        return kwargs.get('format', '{base}/{api_version}/{path}').format(**kwargs)

    # Should be public, it is conflict with memsource endpoint.
    def _post(
            self,
            path: {'Send request to this path': str},
            params: {'Send request with this parameters': dict},
            files: {'Upload this files. Key is filename, value is file object': dict}={},
            timeout: {
                'When takes over this time in one request, raise timeout': (int, float)
            }=constants.Base.timeout.value
    ) -> {'Reponse body': dict}:
        """
        If you want to raw response, you can use _get_stream method.
        TODO: implements _post_stream.
        """
        return self._request(constants.HttpMethod.post, path, files, params, timeout)

    def _get_stream(
            self,
            path: {'Send request to this path': str},
            params: {'Send request with this parameters': dict},
            files: {'Upload this files. Key is filename, value is file object': dict}={},
            timeout: {
                'When takes over this time in one request, raise timeout': (int, float)
            }=constants.Base.timeout.value * 5
    ) -> 'Response object of Requests library':
        """
        This method returns response object of requests library,
        because XML parse or save to file or etc are different will call different method.

        We can switch response by value of stream, but it is not good, I think,
        because type of returning value is only one is easy to use, easy to understand.
        """
        return self._request_stream(constants.HttpMethod.get, path, files, params, timeout)

    def _pre_request(
            self,
            path: {'{api_name}/{method}': str},
            params: {'Insert token in this dict.': dict}
    ) -> (str, dict):
        """
        Create request url and extend param with token for authentication.
        """
        url = self._make_url(
            base=constants.Base.url.value,
            api_version=self.api_version.value,
            path=path
        )
        params['token'] = self.token

        self.last_url = url
        self.last_params = params

        return (url, params, )

    def _request_stream(
            self,
            http_method: {constants.HttpMethod, 'Use this http method'},
            path: {'Send request with this parameters': dict},
            files: {'Upload this files. Key is filename, value is file object': dict},
            params: {'Send request with this parameters': dict},
            timeout: {'When takes over this time in one request, raise timeout': (int, float)}
    ) -> requests.models.Response:
        (url, params_with_token) = self._pre_request(path, params)

        return self._get_response(
            http_method, url, params=params_with_token, files=files, timeout=timeout, stream=True)

    def _get_response(
            self,
            http_method: {'Use this http method': constants.HttpMethod},
            url: {'access to this url': str},
            **kwargs
    ) -> requests.models.Response:
        try:
            response = requests.request(http_method.value, url, **kwargs)
        except requests.exceptions.Timeout:
            raise exceptions.MemsourceApiException(None, {
                'errorCode': 'Internal',
                'errorDescription': 'The request timed out, timeout is {}'.format(
                    kwargs['timeout'] if 'timeout' in kwargs else 'default'),
            }, self.last_url, self.last_params)
        except requests.exceptions.ConnectionError:
            raise exceptions.MemsourceApiException(None, {
                'errorCode': 'Internal',
                'errorDescription': 'Could not connect',
            }, self.last_url, self.last_params)

        if BaseApi.is_success(response.status_code):
            return response

        raise exceptions.MemsourceApiException(
            response.status_code, response.json(), self.last_url, self.last_params)

    def _request(
            self,
            http_method: {constants.HttpMethod, 'Use this http method'},
            path: {'Send request with this parameters': dict},
            files: {'Upload this files. Key is filename, value is file object': dict},
            params: {'Send request with this parameters': dict},
            timeout: {'When takes over this time in one request, raise timeout': (int, float)}
    ) -> {'Parsed esponse body as JSON': dict}:
        (url, params_with_token) = self._pre_request(path, params)

        # If it is successful, returns response json
        return self._get_response(
            http_method, url, params=params_with_token, files=files, timeout=timeout).json()

    @staticmethod
    def is_success(status_code: {int}):
        # if status_code is float type, we will get unexpected result.
        # but I think it is not big deal.
        return 200 <= status_code < 300


class Auth(BaseApi):
    """
    You can see the document http://wiki.memsource.com/wiki/Authentication_API_v3
    """
    api_version = constants.ApiVersion.v3

    def __init__(self, token=None):
        super(Auth, self).__init__(token)

    def login(self, user_name, password):
        r = self._post('auth/login', {
            'userName': user_name,
            'password': password,
        })

        r['user'] = models.User(r.pop('user'))

        return models.Authentication(r)


class Client(BaseApi):
    """
    You can see the document http://wiki.memsource.com/wiki/Client_API_v2
    """
    api_version = constants.ApiVersion.v2

    def create(self, name):
        return self._post('client/create', {
            'name': name,
        })['id']

    def get(self, client):
        return models.Client(self._post('client/get', {
            'client': client,
        }))

    def list(self, page=0):
        return [
            models.Client(client) for client in self._post('client/list', {
                'page': page,
            })
        ]


class Domain(BaseApi):
    """
    You can see the document http://wiki.memsource.com/wiki/Domain_API_v2
    """
    api_version = constants.ApiVersion.v2

    def create(self, name):
        return self._post('domain/create', {
            'name': name,
        })

    def get(self, domain):
        return models.Domain(self._post('domain/get', {
            'domain': domain,
        }))

    def list(self, page=0):
        return [
            models.Domain(domain) for domain in self._post('domain/list', {
                'page': page,
            })
        ]


class Project(BaseApi):
    """
    You can see the document http://wiki.memsource.com/wiki/Project_API_v3
    """
    api_version = constants.ApiVersion.v3

    def create(
            self,
            name: {'project name': str},
            source_lang: str,
            target_langs: {'You can use list for multi target_lang.': (list, tuple, str)},
            client: int=None,
            domain: int=None
    ) -> {'project id', int}:
        """
        Create new project.
        """
        return self._post('project/create', {
            'token': self.token,
            'name': name,
            'sourceLang': source_lang,
            'targetLang': target_langs,
            'client': client,
            'domain': domain,
        })['id']

    def list(self):
        return [models.Project(project) for project in self._post('project/list', {})]

    def getTransMemories(self, project_id):
        return [
            models.TranslationMemory(translation_memory)
            for translation_memory in self._post('project/getTransMemories', {
                'project': project_id
            })
        ]

    def setTransMemories(self, project_id, read_trans_memory_ids=[], write_trans_memory_id=None,
                         penalties=[], target_lang=None):
        """
        You can set translation memory to a project.

        :param project_id :set translation memory to this id of project :int
        :param read_trans_memory_ids :Read these translation memories :[int]
        :param write_trans_memory_id :Write to this translation memory.
        write translation memory must be included in the read translation memories, too :int
        :param penalties :a list of penalties for each read translation memory :[double]
        :param target_lang :set translation memories only for the specific project target language
        :str

        :return :None
        """
        params = {
            'project': project_id,
        }

        # Check the parameters. If the parameter is None or empty list, we ignore it.
        if len(read_trans_memory_ids) != 0:
            params['readTransMemory'] = read_trans_memory_ids

        if write_trans_memory_id is not None:
            params['writeTransMemory'] = write_trans_memory_id

        if len(penalties) != 0:
            params['penalty'] = penalties

        if target_lang is not None:
            params['targetLang'] = target_lang

        # This end-point return nothing.
        self._post('project/setTransMemories', params)

        return None


class Job(BaseApi):
    """
    You can see the document http://wiki.memsource.com/wiki/Job_API_v6
    """
    api_version = constants.ApiVersion.v6

    def create(self, project_id: int, file_path: str, target_langs):
        """
        return: [JobPart]

        If returning JSON has `unsupportedFiles`,
        this method raise MemsourceUnsupportedFileException
        """
        with open(file_path, 'r') as f:
            return self._create(project_id, target_langs, {
                'file': f,
            })

    def createFromText(self, project_id: int, text: str, target_langs, file_name=None):
        """
        You can create a job without a file.
        See: Job.create

        Create file name by uuid1() when file_name parameter is None.
        """
        return self._create(project_id, target_langs, {
            'file': ('{}.txt'.format(uuid.uuid1().hex) if file_name is None else file_name, text),
        })

    def _create(self, project_id: int, target_langs, files: dict):
        result = self._post('job/create', {
            'project': project_id,
            'targetLang': target_langs,
        }, files)

        # unsupported file count is 0 mean success.
        unsupported_files = result.get('unsupportedFiles', [])
        if len(unsupported_files) == 0:
            return [models.JobPart(job_parts) for job_parts in result['jobParts']]

        _, value = files.popitem()
        if isinstance(value, tuple):
            # If value is tuple type, this function called from createFromText.
            # We need to create temporary file for to raise exception.
            file_name, text = value
            file_path = os.path.join('/', 'tmp', file_name)
            with open(file_path, 'w+') as f:
                f.write(text)
        else:
            file_path = value.name

        raise exceptions.MemsourceUnsupportedFileException(
            unsupported_files,
            file_path,
            self.last_url,
            self.last_params
        )

    def listByProject(self, project_id):
        # TODO: wrap inner project
        return [models.JobPart(job_part) for job_part in self._post('job/listByProject', {
            'project': project_id
        })]

    def preTranslate(self, job_parts):
        """
        This API takes long time. you might timed out. You can use Asynchronous.preTranslate
        """
        self._post('job/preTranslate', {'jobPart': job_parts})

    def _getBilingualStream(
            self,
            job_parts: {'Lsit of job_part id': (list, tuple)},
    ) -> types.GeneratorType:
        """
        Common process of bilingualFile.
        """
        return self._get_stream('job/getBilingualFile', {
            'jobPart': job_parts,
        }).iter_content(1024)

    def getBilingualFileXml(self, job_parts: {'Lsit of job_part id': (list, tuple)}) -> bytes:
        buffer = io.BytesIO()
        [buffer.write(chunk) for chunk in self._getBilingualStream(job_parts)]

        return buffer.getvalue()

    def getBilingualFile(
            self,
            job_parts: {'Lsit of job_part id': (list, tuple)},
            dest_file_path: {'Save XML to this file path': str}
    ) -> None:
        """
        Get bilingual file and save it as file.
        """
        with open(dest_file_path, 'wb') as f:
            [f.write(chunk) for chunk in self._getBilingualStream(job_parts)]

    def getBilingualAsMxliffUnits(
            self, job_parts: {'Lsit of job_part id': (list, tuple)},
    ) -> models.MxliffUnit:
        """
        Get bilingual file and parse it as [models.MxliffUnit]
        """
        return mxliff.MxliffParser().parse(self.getBilingualFileXml(job_parts))

    def getSegments(self, task, begin_index, end_index):<commit_msg>[#87901500] Create get completed file function<commit_after>import requests
import uuid
import io
import types
import os.path

from . import constants, exceptions, models
from .lib import mxliff


__all__ = ('Auth', 'Client', 'Domain', 'Project', 'Job', 'TranslationMemory', 'Asynchronous',
           'Analysis')


class BaseApi(object):
    """Inheriting classes must have the api_version attribute"""
    def __init__(self, token: {'Authentication token for using APIs': str}):
        if not hasattr(self, 'api_version'):
            # This exception is for development this library.
            raise NotImplementedError(
                'api_version is not set in {}'.format(self.__class__.__name__))

        self.token = token

    def _make_url(self, *args, **kwargs):
        return kwargs.get('format', '{base}/{api_version}/{path}').format(**kwargs)

    # Should be public, it is conflict with memsource endpoint.
    def _post(
            self,
            path: {'Send request to this path': str},
            params: {'Send request with this parameters': dict},
            files: {'Upload this files. Key is filename, value is file object': dict}={},
            timeout: {
                'When takes over this time in one request, raise timeout': (int, float)
            }=constants.Base.timeout.value
    ) -> {'Reponse body': dict}:
        """
        If you want to raw response, you can use _get_stream method.
        TODO: implements _post_stream.
        """
        return self._request(constants.HttpMethod.post, path, files, params, timeout)

    def _get_stream(
            self,
            path: {'Send request to this path': str},
            params: {'Send request with this parameters': dict},
            files: {'Upload this files. Key is filename, value is file object': dict}={},
            timeout: {
                'When takes over this time in one request, raise timeout': (int, float)
            }=constants.Base.timeout.value * 5
    ) -> 'Response object of Requests library':
        """
        This method returns response object of requests library,
        because XML parse or save to file or etc are different will call different method.

        We can switch response by value of stream, but it is not good, I think,
        because type of returning value is only one is easy to use, easy to understand.
        """
        return self._request_stream(constants.HttpMethod.get, path, files, params, timeout)

    def _pre_request(
            self,
            path: {'{api_name}/{method}': str},
            params: {'Insert token in this dict.': dict}
    ) -> (str, dict):
        """
        Create request url and extend param with token for authentication.
        """
        url = self._make_url(
            base=constants.Base.url.value,
            api_version=self.api_version.value,
            path=path
        )
        params['token'] = self.token

        self.last_url = url
        self.last_params = params

        return (url, params, )

    def _request_stream(
            self,
            http_method: {constants.HttpMethod, 'Use this http method'},
            path: {'Send request with this parameters': dict},
            files: {'Upload this files. Key is filename, value is file object': dict},
            params: {'Send request with this parameters': dict},
            timeout: {'When takes over this time in one request, raise timeout': (int, float)}
    ) -> requests.models.Response:
        (url, params_with_token) = self._pre_request(path, params)

        return self._get_response(
            http_method, url, params=params_with_token, files=files, timeout=timeout, stream=True)

    def _get_response(
            self,
            http_method: {'Use this http method': constants.HttpMethod},
            url: {'access to this url': str},
            **kwargs
    ) -> requests.models.Response:
        try:
            response = requests.request(http_method.value, url, **kwargs)
        except requests.exceptions.Timeout:
            raise exceptions.MemsourceApiException(None, {
                'errorCode': 'Internal',
                'errorDescription': 'The request timed out, timeout is {}'.format(
                    kwargs['timeout'] if 'timeout' in kwargs else 'default'),
            }, self.last_url, self.last_params)
        except requests.exceptions.ConnectionError:
            raise exceptions.MemsourceApiException(None, {
                'errorCode': 'Internal',
                'errorDescription': 'Could not connect',
            }, self.last_url, self.last_params)

        if BaseApi.is_success(response.status_code):
            return response

        raise exceptions.MemsourceApiException(
            response.status_code, response.json(), self.last_url, self.last_params)

    def _request(
            self,
            http_method: {constants.HttpMethod, 'Use this http method'},
            path: {'Send request with this parameters': dict},
            files: {'Upload this files. Key is filename, value is file object': dict},
            params: {'Send request with this parameters': dict},
            timeout: {'When takes over this time in one request, raise timeout': (int, float)}
    ) -> {'Parsed esponse body as JSON': dict}:
        (url, params_with_token) = self._pre_request(path, params)

        # If it is successful, returns response json
        return self._get_response(
            http_method, url, params=params_with_token, files=files, timeout=timeout).json()

    @staticmethod
    def is_success(status_code: {int}):
        # if status_code is float type, we will get unexpected result.
        # but I think it is not big deal.
        return 200 <= status_code < 300


class Auth(BaseApi):
    """
    You can see the document http://wiki.memsource.com/wiki/Authentication_API_v3
    """
    api_version = constants.ApiVersion.v3

    def __init__(self, token=None):
        super(Auth, self).__init__(token)

    def login(self, user_name, password):
        r = self._post('auth/login', {
            'userName': user_name,
            'password': password,
        })

        r['user'] = models.User(r.pop('user'))

        return models.Authentication(r)


class Client(BaseApi):
    """
    You can see the document http://wiki.memsource.com/wiki/Client_API_v2
    """
    api_version = constants.ApiVersion.v2

    def create(self, name):
        return self._post('client/create', {
            'name': name,
        })['id']

    def get(self, client):
        return models.Client(self._post('client/get', {
            'client': client,
        }))

    def list(self, page=0):
        return [
            models.Client(client) for client in self._post('client/list', {
                'page': page,
            })
        ]


class Domain(BaseApi):
    """
    You can see the document http://wiki.memsource.com/wiki/Domain_API_v2
    """
    api_version = constants.ApiVersion.v2

    def create(self, name):
        return self._post('domain/create', {
            'name': name,
        })

    def get(self, domain):
        return models.Domain(self._post('domain/get', {
            'domain': domain,
        }))

    def list(self, page=0):
        return [
            models.Domain(domain) for domain in self._post('domain/list', {
                'page': page,
            })
        ]


class Project(BaseApi):
    """
    You can see the document http://wiki.memsource.com/wiki/Project_API_v3
    """
    api_version = constants.ApiVersion.v3

    def create(
            self,
            name: {'project name': str},
            source_lang: str,
            target_langs: {'You can use list for multi target_lang.': (list, tuple, str)},
            client: int=None,
            domain: int=None
    ) -> {'project id', int}:
        """
        Create new project.
        """
        return self._post('project/create', {
            'token': self.token,
            'name': name,
            'sourceLang': source_lang,
            'targetLang': target_langs,
            'client': client,
            'domain': domain,
        })['id']

    def list(self):
        return [models.Project(project) for project in self._post('project/list', {})]

    def getTransMemories(self, project_id):
        return [
            models.TranslationMemory(translation_memory)
            for translation_memory in self._post('project/getTransMemories', {
                'project': project_id
            })
        ]

    def setTransMemories(self, project_id, read_trans_memory_ids=[], write_trans_memory_id=None,
                         penalties=[], target_lang=None):
        """
        You can set translation memory to a project.

        :param project_id :set translation memory to this id of project :int
        :param read_trans_memory_ids :Read these translation memories :[int]
        :param write_trans_memory_id :Write to this translation memory.
        write translation memory must be included in the read translation memories, too :int
        :param penalties :a list of penalties for each read translation memory :[double]
        :param target_lang :set translation memories only for the specific project target language
        :str

        :return :None
        """
        params = {
            'project': project_id,
        }

        # Check the parameters. If the parameter is None or empty list, we ignore it.
        if len(read_trans_memory_ids) != 0:
            params['readTransMemory'] = read_trans_memory_ids

        if write_trans_memory_id is not None:
            params['writeTransMemory'] = write_trans_memory_id

        if len(penalties) != 0:
            params['penalty'] = penalties

        if target_lang is not None:
            params['targetLang'] = target_lang

        # This end-point return nothing.
        self._post('project/setTransMemories', params)

        return None


class Job(BaseApi):
    """
    You can see the document http://wiki.memsource.com/wiki/Job_API_v6
    """
    api_version = constants.ApiVersion.v6

    def create(self, project_id: int, file_path: str, target_langs):
        """
        return: [JobPart]

        If returning JSON has `unsupportedFiles`,
        this method raise MemsourceUnsupportedFileException
        """
        with open(file_path, 'r') as f:
            return self._create(project_id, target_langs, {
                'file': f,
            })

    def createFromText(self, project_id: int, text: str, target_langs, file_name=None):
        """
        You can create a job without a file.
        See: Job.create

        Create file name by uuid1() when file_name parameter is None.
        """
        return self._create(project_id, target_langs, {
            'file': ('{}.txt'.format(uuid.uuid1().hex) if file_name is None else file_name, text),
        })

    def _create(self, project_id: int, target_langs, files: dict):
        result = self._post('job/create', {
            'project': project_id,
            'targetLang': target_langs,
        }, files)

        # unsupported file count is 0 mean success.
        unsupported_files = result.get('unsupportedFiles', [])
        if len(unsupported_files) == 0:
            return [models.JobPart(job_parts) for job_parts in result['jobParts']]

        _, value = files.popitem()
        if isinstance(value, tuple):
            # If value is tuple type, this function called from createFromText.
            # We need to create temporary file for to raise exception.
            file_name, text = value
            file_path = os.path.join('/', 'tmp', file_name)
            with open(file_path, 'w+') as f:
                f.write(text)
        else:
            file_path = value.name

        raise exceptions.MemsourceUnsupportedFileException(
            unsupported_files,
            file_path,
            self.last_url,
            self.last_params
        )

    def listByProject(self, project_id):
        # TODO: wrap inner project
        return [models.JobPart(job_part) for job_part in self._post('job/listByProject', {
            'project': project_id
        })]

    def preTranslate(self, job_parts):
        """
        This API takes long time. you might timed out. You can use Asynchronous.preTranslate
        """
        self._post('job/preTranslate', {'jobPart': job_parts})

    def _getBilingualStream(
            self,
            job_parts: {'Lsit of job_part id': (list, tuple)},
    ) -> types.GeneratorType:
        """
        Common process of bilingualFile.
        """
        return self._get_stream('job/getBilingualFile', {
            'jobPart': job_parts,
        }).iter_content(1024)

    def getBilingualFileXml(self, job_parts: {'Lsit of job_part id': (list, tuple)}) -> bytes:
        buffer = io.BytesIO()
        [buffer.write(chunk) for chunk in self._getBilingualStream(job_parts)]

        return buffer.getvalue()

    def getBilingualFile(
            self,
            job_parts: {'Lsit of job_part id': (list, tuple)},
            dest_file_path: {'Save XML to this file path': str}
    ) -> None:
        """
        Get bilingual file and save it as file.
        """
        with open(dest_file_path, 'wb') as f:
            [f.write(chunk) for chunk in self._getBilingualStream(job_parts)]

    def getCompletedFileText(self, job_parts: list) -> bytes:
        def getCompletedFileStream() -> types.GeneratorType:
            return self._get_stream('job/getCompletedFile', {
                'jobPart': job_parts,
            }).iter_content(1024)

        buffer = io.BytesIO()
        [buffer.write(chunk) for chunk in getCompletedFileStream()]

        return buffer.getvalue()

    def getBilingualAsMxliffUnits(
            self, job_parts: {'Lsit of job_part id': (list, tuple)},
    ) -> models.MxliffUnit:
        """
        Get bilingual file and parse it as [models.MxliffUnit]
        """
        return mxliff.MxliffParser().parse(self.getBilingualFileXml(job_parts))

    def getSegments(self, task, begin_index, end_index):
{'fraction_non_alphanumeric': 0.10703209520375045, 'fraction_numerical': 0.0025243418680129825, 'mean_word_length': 2.1896710374971247, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2707968', 'n_tokens_mistral': 7655, 'n_tokens_neox': 7452, 'n_words': 2262}