<commit_before># Copyright 2017 Google Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Timesketch API client."""
from __future__ import unicode_literals

import json
import uuid

# pylint: disable=wrong-import-order
import bs4
import requests
# pylint: disable=redefined-builtin
from requests.exceptions import ConnectionError
import webbrowser

import altair
# pylint: disable-msg=import-error
from google_auth_oauthlib import flow as googleauth_flow
import google.auth.transport.requests
import pandas
from .definitions import HTTP_STATUS_CODE_20X
from . import importer


def _error_message(response, message=None, error=RuntimeError):
    """Raise an error using error message extracted from response."""
    if not message:
        message = 'Unknown error, with error: '
    soup = bs4.BeautifulSoup(response.text, features='html.parser')
    text = ''
    if soup.p:
        text = soup.p.string
    raise error('{0:s}, with error [{1:d}] {2!s} {3:s}'.format(
        message, response.status_code, response.reason, text))


class TimesketchApi(object):
    """Timesketch API object

    Attributes:
        api_root: The full URL to the server API endpoint.
        session: Authenticated HTTP session.
    """

    DEFAULT_OAUTH_SCOPE = [
        'https://www.googleapis.com/auth/userinfo.email',
        'openid',
        'https://www.googleapis.com/auth/userinfo.profile'
    ]

    DEFAULT_OAUTH_AUTH_URL = 'https://accounts.google.com/o/oauth2/v2/auth'
    DEFAULT_OAUTH_TOKEN_URL = 'https://oauth2.googleapis.com/token'
    DEFAULT_OAUTH_PROVIDER_URL = 'https://www.googleapis.com/oauth2/v1/certs'
    DEFAULT_OAUTH_OOB_URL = 'urn:ietf:wg:oauth:2.0:oob'
    DEFAULT_OAUTH_API_CALLBACK = '/login/api_callback/'

    def __init__(self,
                 host_uri,
                 username,
                 password='',
                 verify=True,
                 client_id='',
                 client_secret='',
                 auth_mode='timesketch'):
        """Initializes the TimesketchApi object.

        Args:
            host_uri: URI to the Timesketch server (https://<server>/).
            username: User username.
            password: <PASSWORD>.
            verify: Verify server SSL certificate.
            client_id: The client ID if OAUTH auth is used.
            client_secret: The OAUTH client secret if OAUTH is used.
            auth_mode: The authentication mode to use. Defaults to 'timesketch'
                Supported values are 'timesketch' (Timesketch login form),
                'http-basic' (HTTP Basic authentication) and oauth.

        Raises:
            ConnectionError: If the Timesketch server is unreachable.
            RuntimeError: If the client is unable to authenticate to the
                backend.
        """
        self._host_uri = host_uri
        self.api_root = '{0:s}/api/v1'.format(host_uri)
        self._credentials = None
        self._flow = None
        try:
            self.session = self._create_session(
                username, password, verify=verify, client_id=client_id,
                client_secret=client_secret, auth_mode=auth_mode)
        except ConnectionError:
            raise ConnectionError('Timesketch server unreachable')
        except RuntimeError as e:
            raise RuntimeError(
                'Unable to connect to server, with error: {0!s}'.format(e))

    def _authenticate_session(self, session, username, password):
        """Post username/password to authenticate the HTTP seesion.

        Args:
            session: Instance of requests.Session.
            username: User username.
            password: <PASSWORD>.
        """
        # Do a POST to the login handler to set up the session cookies
        data = {'username': username, 'password': password}
        session.post('{0:s}/login/'.format(self._host_uri), data=data)

    def _set_csrf_token(self, session):
        """Retrieve CSRF token from the server and append to HTTP headers.

        Args:
            session: Instance of requests.Session.
        """
        # Scrape the CSRF token from the response
        response = session.get(self._host_uri)
        soup = bs4.BeautifulSoup(response.text, features='html.parser')

        tag = soup.find(id='csrf_token')
        csrf_token = None
        if tag:
            csrf_token = tag.get('value')
        else:
            tag = soup.find('meta', attrs={'name': 'csrf-token'})
            if tag:
                csrf_token = tag.attrs.get('content')

        if not csrf_token:
            return

        session.headers.update({
            'x-csrftoken': csrf_token,
            'referer': self._host_uri
        })

    def _create_oauth_session(self, client_id, client_secret, skip_open=False):
        """Return an OAuth session.

        Args:
            client_id: The client ID if OAUTH auth is used.
            client_secret: The OAUTH client secret if OAUTH is used.
            skip_open: If set to True then an auth URL will be printed, instead
                of presenting an option to open a browser window.

        Return:
            session: Instance of requests.Session.

        Raises:
            RuntimeError: if unable to log in to the application.
        """
        client_config = {
            'installed': {
                'client_id': client_id,
                'client_secret': client_secret,
                'auth_uri': self.DEFAULT_OAUTH_AUTH_URL,
                'token_uri': self.DEFAULT_OAUTH_TOKEN_URL,
                'auth_provider_x509_cert_url': self.DEFAULT_OAUTH_PROVIDER_URL,
                'redirect_uris': [self.DEFAULT_OAUTH_OOB_URL],
            },
        }

        flow = googleauth_flow.InstalledAppFlow.from_client_config(
            client_config, self.DEFAULT_OAUTH_SCOPE)
        flow.redirect_uri = self.DEFAULT_OAUTH_OOB_URL
        auth_url, _ = flow.authorization_url(prompt='select_account')

        if skip_open:
            print('Visit the following URL to authenticate: {0:s}'.format(
                auth_url))
        else:
            open_browser = input('Open the URL in a browser window? [y/N] ')
            if open_browser.lower() == 'y' or open_browser.lower() == 'yes':
                webbrowser.open(auth_url)
            else:
                print(
                    'Need to manually visit URL to authenticate: {0:s}'.format(
                        auth_url))

        code = input('Enter the token code: ')

        _ = flow.fetch_token(code=code)
        session = flow.authorized_session()
        self._flow = flow
        self._credentials = flow.credentials
<commit_msg>Adding a server to catch OAUTH flow.<commit_after># Copyright 2017 Google Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Timesketch API client."""
from __future__ import unicode_literals

import json
import uuid

# pylint: disable=wrong-import-order
import bs4
import requests
# pylint: disable=redefined-builtin
from requests.exceptions import ConnectionError
import webbrowser

import altair
# pylint: disable-msg=import-error
from google_auth_oauthlib import flow as googleauth_flow
import google.auth.transport.requests
import pandas
from .definitions import HTTP_STATUS_CODE_20X
from . import importer


def _error_message(response, message=None, error=RuntimeError):
    """Raise an error using error message extracted from response."""
    if not message:
        message = 'Unknown error, with error: '
    soup = bs4.BeautifulSoup(response.text, features='html.parser')
    text = ''
    if soup.p:
        text = soup.p.string
    raise error('{0:s}, with error [{1:d}] {2!s} {3:s}'.format(
        message, response.status_code, response.reason, text))


class TimesketchApi(object):
    """Timesketch API object

    Attributes:
        api_root: The full URL to the server API endpoint.
        session: Authenticated HTTP session.
    """

    DEFAULT_OAUTH_SCOPE = [
        'https://www.googleapis.com/auth/userinfo.email',
        'openid',
        'https://www.googleapis.com/auth/userinfo.profile'
    ]

    DEFAULT_OAUTH_AUTH_URL = 'https://accounts.google.com/o/oauth2/v2/auth'
    DEFAULT_OAUTH_TOKEN_URL = 'https://oauth2.googleapis.com/token'
    DEFAULT_OAUTH_PROVIDER_URL = 'https://www.googleapis.com/oauth2/v1/certs'
    DEFAULT_OAUTH_OOB_URL = 'urn:ietf:wg:oauth:2.0:oob'
    DEFAULT_OAUTH_API_CALLBACK = '/login/api_callback/'

    def __init__(self,
                 host_uri,
                 username,
                 password='',
                 verify=True,
                 client_id='',
                 client_secret='',
                 auth_mode='timesketch'):
        """Initializes the TimesketchApi object.

        Args:
            host_uri: URI to the Timesketch server (https://<server>/).
            username: User username.
            password: <PASSWORD>.
            verify: Verify server SSL certificate.
            client_id: The client ID if OAUTH auth is used.
            client_secret: The OAUTH client secret if OAUTH is used.
            auth_mode: The authentication mode to use. Defaults to 'timesketch'
                Supported values are 'timesketch' (Timesketch login form),
                'http-basic' (HTTP Basic authentication) and oauth.

        Raises:
            ConnectionError: If the Timesketch server is unreachable.
            RuntimeError: If the client is unable to authenticate to the
                backend.
        """
        self._host_uri = host_uri
        self.api_root = '{0:s}/api/v1'.format(host_uri)
        self._credentials = None
        self._flow = None
        try:
            self.session = self._create_session(
                username, password, verify=verify, client_id=client_id,
                client_secret=client_secret, auth_mode=auth_mode)
        except ConnectionError:
            raise ConnectionError('Timesketch server unreachable')
        except RuntimeError as e:
            raise RuntimeError(
                'Unable to connect to server, with error: {0!s}'.format(e))

    def _authenticate_session(self, session, username, password):
        """Post username/password to authenticate the HTTP seesion.

        Args:
            session: Instance of requests.Session.
            username: User username.
            password: <PASSWORD>.
        """
        # Do a POST to the login handler to set up the session cookies
        data = {'username': username, 'password': password}
        session.post('{0:s}/login/'.format(self._host_uri), data=data)

    def _set_csrf_token(self, session):
        """Retrieve CSRF token from the server and append to HTTP headers.

        Args:
            session: Instance of requests.Session.
        """
        # Scrape the CSRF token from the response
        response = session.get(self._host_uri)
        soup = bs4.BeautifulSoup(response.text, features='html.parser')

        tag = soup.find(id='csrf_token')
        csrf_token = None
        if tag:
            csrf_token = tag.get('value')
        else:
            tag = soup.find('meta', attrs={'name': 'csrf-token'})
            if tag:
                csrf_token = tag.attrs.get('content')

        if not csrf_token:
            return

        session.headers.update({
            'x-csrftoken': csrf_token,
            'referer': self._host_uri
        })

    def _create_oauth_session(
            self, client_id='', client_secret='', client_secrets_file=None,
            run_server=True, skip_open=False):
        """Return an OAuth session.

        Args:
            client_id: The client ID if OAUTH auth is used.
            client_secret: The OAUTH client secret if OAUTH is used.
            client_secrets_file: Path to the JSON file that contains the client
                secrets, in the client_secrets format.
            run_server: A boolean, if set to true (default) a web server is
                run to catch the OAUTH request and response.
            skip_open: A booelan, if set to True (defaults to False) an
                authorization URL is printed on the screen to visit. This is
                only valid if run_server is set to False.

        Return:
            session: Instance of requests.Session.

        Raises:
            RuntimeError: if unable to log in to the application.
        """
        if client_secrets_file:
            if not os.path.isfile(client_secrets_file):
                raise RuntimeError(
                    'Unable to log in, client secret files does not exist.')
            flow = googleauth_flow.InstalledAppFlow.from_client_secrets_file(
                client_secrets_file, scopes=self.DEFAULT_OAUTH_SCOPE)
        else:
            provider_url = self.DEFAULT_OAUTH_PROVIDER_URL
            client_config = {
                'installed': {
                    'client_id': client_id,
                    'client_secret': client_secret,
                    'auth_uri': self.DEFAULT_OAUTH_AUTH_URL,
                    'token_uri': self.DEFAULT_OAUTH_TOKEN_URL,
                    'auth_provider_x509_cert_url': provider_url,
                    'redirect_uris': [self.DEFAULT_OAUTH_OOB_URL],
                },
            }

            flow = googleauth_flow.InstalledAppFlow.from_client_config(
                client_config, self.DEFAULT_OAUTH_SCOPE)

            flow.redirect_uri = self.DEFAULT_OAUTH_OOB_URL

        if run_server:
            _ = flow.run_local_server()
        else:
            auth_url, _ = flow.authorization_url(prompt='select_account')

            if skip_open:
                print('Visit the following URL to authenticate: {0:s}'.format(
                    auth_url))
            else:
                open_browser = input('Open the URL in a browser window? [y/N] ')
                if open_browser.lower() == 'y' or open_browser.lower() == 'yes':
                    webbrowser.open(auth_url)
                else:
                    print(
                        'Need to manually visit URL to authenticate: '
                        '{0:s}'.format(auth_url))

            code = input('Enter the token code: ')
            _ = flow.fetch_token(code=code)

        session = flow.authorized_session()
        self._flow = flow
        self._credentials = flow.credentials

{'fraction_non_alphanumeric': 0.0873202614379085, 'fraction_numerical': 0.004313725490196078, 'mean_word_length': 2.1784378894889906, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 12, 'lorem ipsum': 0, 'www.': 8, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5318600', 'n_tokens_mistral': 4318, 'n_tokens_neox': 4140, 'n_words': 1215}