<commit_before># Copyright 2020 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import base64
import datetime
import struct
import pycountry
from unfurl.parsers.proto.google_search_pb2 import Ved
from google.protobuf import json_format

google_edge = {
    'color': {
        'color': 'green'
    },
    'title': 'Google-related Parsing Functions',
    'label': 'G'
}

known_sources = {
    'hp': 'Home Page',
    'serp': 'Search Engine Results Page',
    'img': 'Image Search',
    'psy-ab': 'Address Bar',
    'hangouts': 'Hangouts Chat'
}


def decode_varint(source):
    result = 0
    number_of_bytes = 0
    for read in source:
        result |= ((read & 0x7F) << (number_of_bytes * 7))
        number_of_bytes += 1
        if (read & 0x80) != 0x80:
            return result, number_of_bytes


def parse_ei(ei):
    """Parse ei parameters from Google searches.

    Based on work by:
      <NAME> (<EMAIL>) -
        https://github.com/cheeky4n6monkey/4n6-scripts/blob/master/google-ei-time.py
      <NAME> - https://deedpolloffice.com/blog/articles/decoding-ei-parameter
    """

    decoded = base64.urlsafe_b64decode(ei)

    # grab 1st 4 bytes and treat as LE unsigned int
    timestamp = struct.unpack('<i', decoded[0:4])[0]
    parsed = [timestamp]

    varint_offset = 4  # First 4 (0-3) bytes are the timestamp
    for _ in [1, 2, 3]:
        value, bytes_used = decode_varint(decoded[varint_offset:])
        parsed.append(value)
        varint_offset += bytes_used

    return parsed


def parse_rlz(rlz_string):
    # Reference: https://github.com/rogerta/rlz/blob/wiki/HowToReadAnRlzString.md

    rlz = {}
    assert rlz_string[0] == '1', 'Only v1 of RLZ is known'

    rlz['version'] = {
            'data_type': 'google.rlz.version',
            'key': 'RLZ version',
            'value': rlz_string[0],
            'hover': 'Only v1 of RLZ is known'}

    rlz['ap'] = {
        'data_type': 'google.rlz.ap',
        'key': 'Application',
        'value': rlz_string[1:3],
        'hover': 'Application (or "access point") used to do the Google Search'}

    rlz['brand_code'] = {
        'data_type': 'google.rlz.brand_code',
        'key': 'Brand Code',
        'value': rlz_string[3:7],
        'hover': 'The brand code identifies the distribution channel (it may be a partner or internal marketing). <br>'
                 'This correlates to how the user got the software (ie. they downloaded it by itself vs. it came <br>'
                 'pre-installed on their new computer vs. it came with a partner\'s software.'}

    cannibal_string = rlz_string[7]
    if cannibal_string == 'c':
        cannibal_value = 'Yes'
    elif cannibal_string == '_':
        cannibal_value = 'No'
    else:
        raise ValueError(f'Invalid "cannibal" value in RLZ string: {cannibal_string}')

    rlz['cannibal'] = {
        'data_type': 'google.rlz.cannibal',
        'key': 'Cannibalized',
        'value': cannibal_value,
        'hover': '"Cannibal" tells if the library has evidence that the user was a user prior <br>'
                 'to installing the software, and thus "cannibalized" a previous installation.'}

    rlz_pointer = 8

    # Examples of longer RLZ values (2- and 5-char lang codes):
    #   1C1CHBF_en-GBGB901GB901
    #   1C1GCEU_enUS820US820
    if len(rlz_string) >= 11:
        # If a dash is in this position, it means it is a 5 char lang code, not the 2 char code
        if rlz_string[10] == '-':
            language_code = rlz_string[rlz_pointer:rlz_pointer+5]
            rlz_pointer += 5
            # langcodes was having install issues on macOS; not using it for now in
            # order to not complicate Unfurl's install. Pycountry's languages isn't
            # as good (only alpha_2 and alpha_3) but better than nothing for now.
            # Old implementation:
            # language_name = langcodes.Language.get(language_code).language_name()
            language_name = pycountry.languages.get(alpha_2=language_code[:2]).name

        else:
            language_code = rlz_string[rlz_pointer:rlz_pointer+2]
            # language_name = langcodes.Language.get(language_code).language_name()
            language_name = pycountry.languages.get(alpha_2=language_code).name
            rlz_pointer += 2

    # Example of RLZ value without cohorts
    #   1C1GCEV_en
    elif len(rlz_string) == 10:
        language_code = rlz_string[rlz_pointer:rlz_pointer + 2]
        # language_name = langcodes.Language.get(language_code).language_name()
        language_name = pycountry.languages.get(alpha_2=language_code).name
        rlz_pointer += 2

    else:
        raise ValueError('Couldn\'t parse RLZ language code')

    rlz['language'] = {
        'data_type': 'google.rlz.language',
        'key': 'Language',
        'value': f'{language_name} ({language_code})',
        'hover': 'The two- (en) or five-character (zh-CN) language code of the application. <br>'
                 'Valid values depend on the specific app.'}

    # The cohorts are optional, so if this is the end of the RLZ string, return what we've parsed
    if not len(rlz_string) >= rlz_pointer+5:
        return rlz

    install_cohort = rlz_string[rlz_pointer:rlz_pointer+5]
    rlz_pointer += 5

    install_country_code = install_cohort[:2]
    install_country = pycountry.countries.get(alpha_2=install_country_code).name
    install_date = datetime.timedelta(weeks=int(install_cohort[2:])) + datetime.date(year=2003, month=2, day=3)

    rlz['install_cohort'] = {
        'data_type': 'google.rlz.install_cohort',
        'key': 'Install Cohort',
        'value': f'Installed in {install_country} the week of {install_date}',
        'hover': 'The country and week of the user\'s installation event. <br>'
                 'Country is determined by the server, using IP address. <br>'
                 'Week is measured as number of weeks since Feb 3, 2003.'}

    # Both cohorts do not need to be present, so if the 2nd one (search) is missing, return what we have
    # Example: 1CAGZLV_enGB898
    if not len(rlz_string) >= rlz_pointer + 5:
        return rlz

    search_cohort = rlz_string[rlz_pointer:rlz_pointer + 5]

    search_country_code = search_cohort[:2]
    search_country = pycountry.countries.get(alpha_2=search_country_code).name
    search_date = datetime.timedelta(weeks=int(search_cohort[2:])) + datetime.date(year=2003, month=2, day=3)

    rlz['search_cohort'] = {
        'data_type': 'google.rlz.search_cohort',
        'key': 'Search Cohort',
        'value': f'First search was in {search_country} the week of {search_date}',
        'hover': 'The country and week of the user\'s first search. <br>'
                 'Country is determined by the server, using IP address. <br>'
                 'Week is measured as number of weeks since Feb 3, 2003.'}

    return rlz


def run(unfurl, node):
    if node.data_type == 'url.query.pair':
        if 'google' in unfurl.find_preceding_domain(node):
            if node.key == 'ei':
                parsed_ei = parse_ei(unfurl.add_b64_padding(node.value))
                node.hover = 'The \'<b>ei</b>\' parameter is base64-encoded and contains four values. ' \
                             '<br>The first two are thought to be the timestamp of when the session started ' \
                             '<br>(first value is full seconds, second is microsecond component)' \
                             '<br>This may be seconds (but could be days!) before the URL was generated.' \
                             '<br><br>References:<ul>' \
                             '<li><a href="https://deedpolloffice.com/blog/articles/decoding-ei-parameter" ' \
                             'target="_blank"><NAME>: How to decode the ei parameter in Google search</a></li>' \
                             '<li><a href="http://cheeky4n6monkey.blogspot.com/2014/10/google-eid.html" ' \
                             'target="_blank">Cheeky4n6Monkey: Google-ei\'d ?!</a></li>' \
                             '<li><a href="https://github.com/obsidianforensics/unfurl/issues/56" ' \
                             'target="_blank">Rasmus-Riis: Search Experiment with ei parameter</a></li>'\
                             '<li><NAME>: noticed the 2nd ei value contained fractional seconds ' \
                             '<br> that match the ved</li></ul>'
                node.extra_options = {'widthConstraint': {'maximum': 300}}

                assert len(parsed_ei) == 4, \
                    f'There should be 4 decoded ei values, but we have {len(parsed_ei)}!'

                # The first ei value is epoch seconds, the second has the fractional (micro) seconds.
                # Concatenating them here as strings lets the timestamp parser convert it later.
                ei_timestamp = str(parsed_ei[0]) + str(parsed_ei[1])

                unfurl.add_to_queue(
                    data_type='epoch-microseconds', key=None, value=ei_timestamp, label=f'ei Timestamp: {ei_timestamp}',
                    hover='The first two values combined in the <b>ei</b> parameter are thought to be the timestamp of '
                          'when the session began.', parent_id=node.node_id, incoming_edge_config=google_edge)

                unfurl.add_to_queue(
                    data_type='integer', key=None, value=parsed_ei[2],<commit_msg>Add browser window width & height parameters<commit_after># Copyright 2020 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import base64
import datetime
import struct
import pycountry
from unfurl.parsers.proto.google_search_pb2 import Ved
from google.protobuf import json_format

google_edge = {
    'color': {
        'color': 'green'
    },
    'title': 'Google-related Parsing Functions',
    'label': 'G'
}

known_sources = {
    'hp': 'Home Page',
    'serp': 'Search Engine Results Page',
    'img': 'Image Search',
    'psy-ab': 'Address Bar',
    'hangouts': 'Hangouts Chat'
}


def decode_varint(source):
    result = 0
    number_of_bytes = 0
    for read in source:
        result |= ((read & 0x7F) << (number_of_bytes * 7))
        number_of_bytes += 1
        if (read & 0x80) != 0x80:
            return result, number_of_bytes


def parse_ei(ei):
    """Parse ei parameters from Google searches.

    Based on work by:
      <NAME> (<EMAIL>) -
        https://github.com/cheeky4n6monkey/4n6-scripts/blob/master/google-ei-time.py
      <NAME> - https://deedpolloffice.com/blog/articles/decoding-ei-parameter
    """

    decoded = base64.urlsafe_b64decode(ei)

    # grab 1st 4 bytes and treat as LE unsigned int
    timestamp = struct.unpack('<i', decoded[0:4])[0]
    parsed = [timestamp]

    varint_offset = 4  # First 4 (0-3) bytes are the timestamp
    for _ in [1, 2, 3]:
        value, bytes_used = decode_varint(decoded[varint_offset:])
        parsed.append(value)
        varint_offset += bytes_used

    return parsed


def parse_rlz(rlz_string):
    # Reference: https://github.com/rogerta/rlz/blob/wiki/HowToReadAnRlzString.md

    rlz = {}
    assert rlz_string[0] == '1', 'Only v1 of RLZ is known'

    rlz['version'] = {
            'data_type': 'google.rlz.version',
            'key': 'RLZ version',
            'value': rlz_string[0],
            'hover': 'Only v1 of RLZ is known'}

    rlz['ap'] = {
        'data_type': 'google.rlz.ap',
        'key': 'Application',
        'value': rlz_string[1:3],
        'hover': 'Application (or "access point") used to do the Google Search'}

    rlz['brand_code'] = {
        'data_type': 'google.rlz.brand_code',
        'key': 'Brand Code',
        'value': rlz_string[3:7],
        'hover': 'The brand code identifies the distribution channel (it may be a partner or internal marketing). <br>'
                 'This correlates to how the user got the software (ie. they downloaded it by itself vs. it came <br>'
                 'pre-installed on their new computer vs. it came with a partner\'s software.'}

    cannibal_string = rlz_string[7]
    if cannibal_string == 'c':
        cannibal_value = 'Yes'
    elif cannibal_string == '_':
        cannibal_value = 'No'
    else:
        raise ValueError(f'Invalid "cannibal" value in RLZ string: {cannibal_string}')

    rlz['cannibal'] = {
        'data_type': 'google.rlz.cannibal',
        'key': 'Cannibalized',
        'value': cannibal_value,
        'hover': '"Cannibal" tells if the library has evidence that the user was a user prior <br>'
                 'to installing the software, and thus "cannibalized" a previous installation.'}

    rlz_pointer = 8

    # Examples of longer RLZ values (2- and 5-char lang codes):
    #   1C1CHBF_en-GBGB901GB901
    #   1C1GCEU_enUS820US820
    if len(rlz_string) >= 11:
        # If a dash is in this position, it means it is a 5 char lang code, not the 2 char code
        if rlz_string[10] == '-':
            language_code = rlz_string[rlz_pointer:rlz_pointer+5]
            rlz_pointer += 5
            # langcodes was having install issues on macOS; not using it for now in
            # order to not complicate Unfurl's install. Pycountry's languages isn't
            # as good (only alpha_2 and alpha_3) but better than nothing for now.
            # Old implementation:
            # language_name = langcodes.Language.get(language_code).language_name()
            language_name = pycountry.languages.get(alpha_2=language_code[:2]).name

        else:
            language_code = rlz_string[rlz_pointer:rlz_pointer+2]
            # language_name = langcodes.Language.get(language_code).language_name()
            language_name = pycountry.languages.get(alpha_2=language_code).name
            rlz_pointer += 2

    # Example of RLZ value without cohorts
    #   1C1GCEV_en
    elif len(rlz_string) == 10:
        language_code = rlz_string[rlz_pointer:rlz_pointer + 2]
        # language_name = langcodes.Language.get(language_code).language_name()
        language_name = pycountry.languages.get(alpha_2=language_code).name
        rlz_pointer += 2

    else:
        raise ValueError('Couldn\'t parse RLZ language code')

    rlz['language'] = {
        'data_type': 'google.rlz.language',
        'key': 'Language',
        'value': f'{language_name} ({language_code})',
        'hover': 'The two- (en) or five-character (zh-CN) language code of the application. <br>'
                 'Valid values depend on the specific app.'}

    # The cohorts are optional, so if this is the end of the RLZ string, return what we've parsed
    if not len(rlz_string) >= rlz_pointer+5:
        return rlz

    install_cohort = rlz_string[rlz_pointer:rlz_pointer+5]
    rlz_pointer += 5

    install_country_code = install_cohort[:2]
    install_country = pycountry.countries.get(alpha_2=install_country_code).name
    install_date = datetime.timedelta(weeks=int(install_cohort[2:])) + datetime.date(year=2003, month=2, day=3)

    rlz['install_cohort'] = {
        'data_type': 'google.rlz.install_cohort',
        'key': 'Install Cohort',
        'value': f'Installed in {install_country} the week of {install_date}',
        'hover': 'The country and week of the user\'s installation event. <br>'
                 'Country is determined by the server, using IP address. <br>'
                 'Week is measured as number of weeks since Feb 3, 2003.'}

    # Both cohorts do not need to be present, so if the 2nd one (search) is missing, return what we have
    # Example: 1CAGZLV_enGB898
    if not len(rlz_string) >= rlz_pointer + 5:
        return rlz

    search_cohort = rlz_string[rlz_pointer:rlz_pointer + 5]

    search_country_code = search_cohort[:2]
    search_country = pycountry.countries.get(alpha_2=search_country_code).name
    search_date = datetime.timedelta(weeks=int(search_cohort[2:])) + datetime.date(year=2003, month=2, day=3)

    rlz['search_cohort'] = {
        'data_type': 'google.rlz.search_cohort',
        'key': 'Search Cohort',
        'value': f'First search was in {search_country} the week of {search_date}',
        'hover': 'The country and week of the user\'s first search. <br>'
                 'Country is determined by the server, using IP address. <br>'
                 'Week is measured as number of weeks since Feb 3, 2003.'}

    return rlz


def run(unfurl, node):
    if node.data_type == 'url.query.pair':
        if 'google' in unfurl.find_preceding_domain(node):
            if node.key == 'bih':
                unfurl.add_to_queue(
                    data_type='google.bih', key='Browser Height', value=node.value,
                    label=f'Browser height: {node.value}px',
                    hover='Inner height of the browser window',
                    parent_id=node.node_id, incoming_edge_config=google_edge)

            elif node.key == 'biw':
                unfurl.add_to_queue(
                    data_type='google.biw', key='Browser Width', value=node.value,
                    label=f'Browser width: {node.value}px',
                    hover='Inner width of the browser window',
                    parent_id=node.node_id, incoming_edge_config=google_edge)

            elif node.key == 'ei':
                parsed_ei = parse_ei(unfurl.add_b64_padding(node.value))
                node.hover = 'The \'<b>ei</b>\' parameter is base64-encoded and contains four values. ' \
                             '<br>The first two are thought to be the timestamp of when the session started ' \
                             '<br>(first value is full seconds, second is microsecond component)' \
                             '<br>This may be seconds (but could be days!) before the URL was generated.' \
                             '<br><br>References:<ul>' \
                             '<li><a href="https://deedpolloffice.com/blog/articles/decoding-ei-parameter" ' \
                             'target="_blank"><NAME>: How to decode the ei parameter in Google search</a></li>' \
                             '<li><a href="http://cheeky4n6monkey.blogspot.com/2014/10/google-eid.html" ' \
                             'target="_blank">Cheeky4n6Monkey: Google-ei\'d ?!</a></li>' \
                             '<li><a href="https://github.com/obsidianforensics/unfurl/issues/56" ' \
                             'target="_blank">Rasmus-Riis: Search Experiment with ei parameter</a></li>'\
                             '<li><NAME>: noticed the 2nd ei value contained fractional seconds ' \
                             '<br> that match the ved</li></ul>'
                node.extra_options = {'widthConstraint': {'maximum': 300}}

                assert len(parsed_ei) == 4, \
                    f'There should be 4 decoded ei values, but we have {len(parsed_ei)}!'

                # The first ei value is epoch seconds, the second has the fractional (micro) seconds.
                # Concatenating them here as strings lets the timestamp parser convert it later.
                ei_timestamp = str(parsed_ei[0]) + str(parsed_ei[1])

                unfurl.add_to_queue(
                    data_type='epoch-microseconds', key=None, value=ei_timestamp, label=f'ei Timestamp: {ei_timestamp}',
                    hover='The first two values combined in the <b>ei</b> parameter are thought to be the timestamp of '
                          'when the session began.', parent_id=node.node_id, incoming_edge_config=google_edge)

                unfurl.add_to_queue(
                    data_type='integer', key=None, value=parsed_ei[2],
{'fraction_non_alphanumeric': 0.11104048503813808, 'fraction_numerical': 0.015353021709368277, 'mean_word_length': 2.4473285016012136, 'pattern_counts': {'":': 0, '<': 87, '<?xml version=': 0, '>': 87, 'https://': 12, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '903077', 'n_tokens_mistral': 6513, 'n_tokens_neox': 6096, 'n_words': 1772}