<commit_before>import abc
from collections import defaultdict
import glob
import json
import os
import os.path as p
import shutil
import gzip
from flask import safe_join

import menpo.io as mio
from menpo.shape.mesh import TexturedTriMesh
import menpo

from .utils import load_template
from .api import (MeshLandmarkerIOAdapter, LandmarkerIOAdapter,
                  ImageLandmarkerIOAdapter)

CACHE_DIRNAME = 'lmiocache'
TEMPLATE_DINAME = '.lmiotemplates'
TEMPLATE_EXT = '.txt'
TEXTURE_FILENAME = 'texture.jpg'
IMAGE_INFO_FILENAME = 'image.json'
THUMBNAIL_FILENAME = 'thumbnail.jpg'
MESH_FILENAME = 'mesh.json.gz'
LANDMARKS_DIR = 'lmiolandmarks'


def asset_id_for_path(fp):
    return p.splitext(p.split(fp)[-1])[0]


def save_jpg_thumbnail_file(img, path, width=640):
    ip = img.as_PILImage()
    w, h = ip.size
    h2w = h * 1. / w
    ips = ip.resize((width, int(h2w * width)))
    ips.save(path, quality=20, format='jpeg')


class MenpoAdapter(LandmarkerIOAdapter):

    def __init__(self, asset_dir, recursive=False,
                 extension=None, landmark_dir=None, template_dir=None,
                 cache_dir=None):
        # 1. asset dir
        self.asset_dir = p.abspath(p.expanduser(asset_dir))
        if not p.isdir(self.asset_dir):
            raise ValueError('{} is not a directory.'.format(self.asset_dir))
        print ('assets:    {}'.format(self.asset_dir))
        self.recursive = recursive
        if self.recursive:
            print('assets dir will be searched recursively.')
        if extension is not None:
            self.extension_str = '.' + extension
            print('only assets of type {} will be '
                  'loaded.'.format(self.extension_str))
        else:
            self.extension_str = ''
        # 2. landmark dir
        if landmark_dir is None:
            # By default place the landmarks in the cwd
            landmark_dir = p.join(os.getcwd(), LANDMARKS_DIR)
        self.landmark_dir = p.abspath(p.expanduser(landmark_dir))
        if not p.isdir(self.landmark_dir):
            print("Warning the landmark dir does not exist - creating...")
            os.mkdir(self.landmark_dir)

        # 3. template dir
        if template_dir is None:
            # try the user folder
            user_templates = p.expanduser(p.join('~', TEMPLATE_DINAME))
            if p.isdir(user_templates):
                template_dir = user_templates
            else:
                raise ValueError("No template dir provided and "
                                 "{} doesn't exist".format(user_templates))
        self.template_dir = p.abspath(p.expanduser(template_dir))

        # 4. cache dir
        if cache_dir is None:
            # By default place the cache in the cwd
            cache_dir = p.join(os.getcwd(), CACHE_DIRNAME)
        self.cache_dir = p.abspath(p.expanduser(cache_dir))
        if not p.isdir(self.cache_dir):
            print("Warning the cache dir does not exist - creating...")
            os.mkdir(self.cache_dir)
        print ('landmarks: {}'.format(self.landmark_dir))
        print ('templates: {}'.format(self.template_dir))
        print ('cache:     {}'.format(self.cache_dir))

        # Construct a mapping from id's to file paths
        self.asset_paths = {}
        self._build_asset_mapping()

        # Check cache
        asset_ids = set(self.asset_paths.iterkeys())
        cached = set(os.listdir(self.cache_dir))
        uncached = asset_ids - cached
        print('{} assets need to be added to '
              'the cache'.format(len(uncached)))
        for i, asset in enumerate(uncached):
            print('Caching {}/{} - {}'.format(i + 1, len(uncached), asset))
            self.cache_asset(asset)
        if len(uncached) > 0:
                print('{} assets cached.'.format(len(uncached)))

    @abc.abstractproperty
    def n_dims(self):
        pass

    def cache_asset(self, asset_id):
        r"""
        Caches the info for a given asset id so it can be efficiently
        served in the future.

        Parameters
        ----------
        asset_id : `str`
        The id of the asset that needs to be cached
        """
        print('Caching asset {}'.format(asset_id))
        if not asset_id in self.asset_paths:
            self._build_asset_mapping()
        if not asset_id in self.asset_paths:
            raise ValueError('{} is not a valid asset_id'.format(asset_id))
        asset_cache_dir = p.join(self.cache_dir, asset_id)
        if not p.isdir(asset_cache_dir):
            print("Cache for {} does not exist - creating...".format(asset_id))
            os.mkdir(asset_cache_dir)
        self._cache_asset(asset_id)

    @property
    def _glob_pattern(self):
        file_glob = '*' + self.extension_str
        if self.recursive:
            return os.path.join('**', file_glob)
        else:
            return file_glob

    @abc.abstractmethod<commit_msg>no need to check and rebuild asset_mapping<commit_after>import abc
from collections import defaultdict
import glob
import json
import os
import os.path as p
import shutil
import gzip
from flask import safe_join

import menpo.io as mio
from menpo.shape.mesh import TexturedTriMesh
import menpo

from .utils import load_template
from .api import (MeshLandmarkerIOAdapter, LandmarkerIOAdapter,
                  ImageLandmarkerIOAdapter)

CACHE_DIRNAME = 'lmiocache'
TEMPLATE_DINAME = '.lmiotemplates'
TEMPLATE_EXT = '.txt'
TEXTURE_FILENAME = 'texture.jpg'
IMAGE_INFO_FILENAME = 'image.json'
THUMBNAIL_FILENAME = 'thumbnail.jpg'
MESH_FILENAME = 'mesh.json.gz'
LANDMARKS_DIR = 'lmiolandmarks'


def asset_id_for_path(fp):
    return p.splitext(p.split(fp)[-1])[0]


def save_jpg_thumbnail_file(img, path, width=640):
    ip = img.as_PILImage()
    w, h = ip.size
    h2w = h * 1. / w
    ips = ip.resize((width, int(h2w * width)))
    ips.save(path, quality=20, format='jpeg')


class MenpoAdapter(LandmarkerIOAdapter):

    def __init__(self, asset_dir, recursive=False,
                 extension=None, landmark_dir=None, template_dir=None,
                 cache_dir=None):
        # 1. asset dir
        self.asset_dir = p.abspath(p.expanduser(asset_dir))
        if not p.isdir(self.asset_dir):
            raise ValueError('{} is not a directory.'.format(self.asset_dir))
        print ('assets:    {}'.format(self.asset_dir))
        self.recursive = recursive
        if self.recursive:
            print('assets dir will be searched recursively.')
        if extension is not None:
            self.extension_str = '.' + extension
            print('only assets of type {} will be '
                  'loaded.'.format(self.extension_str))
        else:
            self.extension_str = ''
        # 2. landmark dir
        if landmark_dir is None:
            # By default place the landmarks in the cwd
            landmark_dir = p.join(os.getcwd(), LANDMARKS_DIR)
        self.landmark_dir = p.abspath(p.expanduser(landmark_dir))
        if not p.isdir(self.landmark_dir):
            print("Warning the landmark dir does not exist - creating...")
            os.mkdir(self.landmark_dir)

        # 3. template dir
        if template_dir is None:
            # try the user folder
            user_templates = p.expanduser(p.join('~', TEMPLATE_DINAME))
            if p.isdir(user_templates):
                template_dir = user_templates
            else:
                raise ValueError("No template dir provided and "
                                 "{} doesn't exist".format(user_templates))
        self.template_dir = p.abspath(p.expanduser(template_dir))

        # 4. cache dir
        if cache_dir is None:
            # By default place the cache in the cwd
            cache_dir = p.join(os.getcwd(), CACHE_DIRNAME)
        self.cache_dir = p.abspath(p.expanduser(cache_dir))
        if not p.isdir(self.cache_dir):
            print("Warning the cache dir does not exist - creating...")
            os.mkdir(self.cache_dir)
        print ('landmarks: {}'.format(self.landmark_dir))
        print ('templates: {}'.format(self.template_dir))
        print ('cache:     {}'.format(self.cache_dir))

        # Construct a mapping from id's to file paths
        self.asset_paths = {}
        self._build_asset_mapping()

        # Check cache
        asset_ids = set(self.asset_paths.iterkeys())
        cached = set(os.listdir(self.cache_dir))
        uncached = asset_ids - cached
        print('{} assets need to be added to '
              'the cache'.format(len(uncached)))
        for i, asset in enumerate(uncached):
            print('Caching {}/{} - {}'.format(i + 1, len(uncached), asset))
            self.cache_asset(asset)
        if len(uncached) > 0:
                print('{} assets cached.'.format(len(uncached)))

    @abc.abstractproperty
    def n_dims(self):
        pass

    def cache_asset(self, asset_id):
        r"""
        Caches the info for a given asset id so it can be efficiently
        served in the future.

        Parameters
        ----------
        asset_id : `str`
        The id of the asset that needs to be cached
        """
        print('Caching asset {}'.format(asset_id))
        if not asset_id in self.asset_paths:
            raise ValueError('{} is not a valid asset_id'.format(asset_id))
        asset_cache_dir = p.join(self.cache_dir, asset_id)
        if not p.isdir(asset_cache_dir):
            print("Cache for {} does not exist - creating...".format(asset_id))
            os.mkdir(asset_cache_dir)
        self._cache_asset(asset_id)

    @property
    def _glob_pattern(self):
        file_glob = '*' + self.extension_str
        if self.recursive:
            return os.path.join('**', file_glob)
        else:
            return file_glob

    @abc.abstractmethod
{'fraction_non_alphanumeric': 0.10592698933552092, 'fraction_numerical': 0.003281378178835111, 'mean_word_length': 2.2586034079518877, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7130448', 'n_tokens_mistral': 3010, 'n_tokens_neox': 2912, 'n_words': 741}