<commit_before>import os
import json
import hashlib
from boto.s3.key import Key
from splice.models import Tile
from splice.queries import tile_exists, insert_tile
from splice.environment import Environment

env = Environment.instance()


class IngestError(Exception):
    pass


def ingest_links(data, logger=None, *args, **kwargs):
    """
    Obtain links, insert in data warehouse
    """

    ingested_data = {}
    for country_code, tile_data in data.iteritems():
        country_code = country_code.upper()

        if country_code not in env.fixtures["countries"]:
            raise IngestError("ERROR: country_code '{0}' is invalid\n\nvalid countries: {1}".format(country_code, json.dumps(env.fixtures["countries"], indent=2)))
        logger.info("PROCESSING FOR COUNTRY: {0}".format(country_code))

        for locale, tiles in tile_data.iteritems():

            if locale not in env.fixtures["locales"]:
                raise IngestError("ERROR: locale '{0}' is invalid\n\nvalid locales: {1}".format(locale, json.dumps(list(env.fixtures["locales"]), indent=2)))

            new_tiles_data = {}
            new_tiles_list = []

            for t in tiles:
                columns = dict(
                    target_url=t["url"],
                    bg_color=t["bgColor"],
                    title=t["title"],
                    type=t["type"],
                    image_uri=t["imageURI"],
                    enhanced_image_uri=t.get("enhancedImageURI"),
                    locale=locale,
                )

                db_tile_id = tile_exists(**columns)
                f_tile_id = t.get("directoryId")
<commit_msg>store sha1 hash of images instead of the b64 data<commit_after>import os
import json
import hashlib
from boto.s3.key import Key
from splice.models import Tile
from splice.queries import tile_exists, insert_tile
from splice.environment import Environment

env = Environment.instance()


class IngestError(Exception):
    pass


def ingest_links(data, logger=None, *args, **kwargs):
    """
    Obtain links, insert in data warehouse
    """

    ingested_data = {}
    for country_code, tile_data in data.iteritems():
        country_code = country_code.upper()

        if country_code not in env.fixtures["countries"]:
            raise IngestError("ERROR: country_code '{0}' is invalid\n\nvalid countries: {1}".format(country_code, json.dumps(env.fixtures["countries"], indent=2)))
        logger.info("PROCESSING FOR COUNTRY: {0}".format(country_code))

        for locale, tiles in tile_data.iteritems():

            if locale not in env.fixtures["locales"]:
                raise IngestError("ERROR: locale '{0}' is invalid\n\nvalid locales: {1}".format(locale, json.dumps(list(env.fixtures["locales"]), indent=2)))

            new_tiles_data = {}
            new_tiles_list = []

            for t in tiles:
                image_hash = hashlib.sha1(t["imageURI"]).hexdigest()
                enhanced_image_hash = hashlib.sha1(t.get("enhancedImageURI")).hexdigest() if "enhancedImageURI" in t else None

                columns = dict(
                    target_url=t["url"],
                    bg_color=t["bgColor"],
                    title=t["title"],
                    type=t["type"],
                    image_uri=image_hash,
                    enhanced_image_uri=enhanced_image_hash,
                    locale=locale,
                )

                db_tile_id = tile_exists(**columns)
                f_tile_id = t.get("directoryId")

{'fraction_non_alphanumeric': 0.11384439359267734, 'fraction_numerical': 0.006006864988558352, 'mean_word_length': 2.3528283796740173, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7344084', 'n_tokens_mistral': 1040, 'n_tokens_neox': 979, 'n_words': 229}