<commit_before>#!/usr/bin/env python
"""
Commands related to the syncing assets.
"""
from glob import glob
import logging
import os

from fabric.api import prompt, task
import app_config
from fnmatch import fnmatch
import utils

logging.basicConfig(format=app_config.LOG_FORMAT)
logger = logging.getLogger(__name__)
logger.setLevel(app_config.LOG_LEVEL)

ASSETS_ROOT = 'www/assets'

@task
def sync():
    """
    Intelligently synchronize assets between S3 and local folder.
    """
    ignore_globs = []

    with open('%s/assetsignore' % ASSETS_ROOT, 'r') as f:
        ignore_globs = [l.strip() for l in f]

    local_paths = []
    not_lowercase = []

    for local_path, subdirs, filenames in os.walk(ASSETS_ROOT):
        for name in filenames:
            full_path = os.path.join(local_path, name)
            glob_path = full_path.split(ASSETS_ROOT)[1].strip('/')

            ignore = False

            for ignore_glob in ignore_globs:
                if fnmatch(glob_path, ignore_glob):
                    ignore = True
                    break

            if ignore:
                logger.info('Ignoring: %s' % full_path)
                continue

            if name.lower() != name:
                not_lowercase.append(full_path)

            local_paths.append(full_path)

    # Prevent case sensitivity differences between OSX and S3 from screwing us up
    if not_lowercase:
        logger.error('The following filenames are not lowercase, please change them before running `assets.sync`:')

        for name in not_lowercase:
            logger.error('    %s' % name)

        return

    bucket = utils.get_bucket(app_config.ASSETS_S3_BUCKET)
    keys = bucket.list(app_config.ASSETS_SLUG)

    which = None
    always = False

    for key in keys:
        download = False
        upload = False

        local_path = key.name.replace(app_config.ASSETS_SLUG, ASSETS_ROOT, 1)

        # Skip root key
        if local_path == '%s/' % ASSETS_ROOT:
            continue

        logger.info(local_path)

        if local_path in local_paths:
            # A file can only exist once, this speeds up future checks
            # and provides a list of non-existing files when complete
            local_paths.remove(local_path)
<commit_msg>fix assets sync preffix issue<commit_after>#!/usr/bin/env python
"""
Commands related to the syncing assets.
"""
from glob import glob
import logging
import os

from fabric.api import prompt, task
import app_config
from fnmatch import fnmatch
import utils

logging.basicConfig(format=app_config.LOG_FORMAT)
logger = logging.getLogger(__name__)
logger.setLevel(app_config.LOG_LEVEL)

ASSETS_ROOT = 'www/assets'

@task
def sync():
    """
    Intelligently synchronize assets between S3 and local folder.
    """
    ignore_globs = []

    with open('%s/assetsignore' % ASSETS_ROOT, 'r') as f:
        ignore_globs = [l.strip() for l in f]

    local_paths = []
    not_lowercase = []

    for local_path, subdirs, filenames in os.walk(ASSETS_ROOT):
        for name in filenames:
            full_path = os.path.join(local_path, name)
            glob_path = full_path.split(ASSETS_ROOT)[1].strip('/')

            ignore = False

            for ignore_glob in ignore_globs:
                if fnmatch(glob_path, ignore_glob):
                    ignore = True
                    break

            if ignore:
                logger.info('Ignoring: %s' % full_path)
                continue

            if name.lower() != name:
                not_lowercase.append(full_path)

            local_paths.append(full_path)

    # Prevent case sensitivity differences between OSX and S3 from screwing us up
    if not_lowercase:
        logger.error('The following filenames are not lowercase, please change them before running `assets.sync`:')

        for name in not_lowercase:
            logger.error(name)

        return

    bucket = utils.get_bucket(app_config.ASSETS_S3_BUCKET)
    keys = bucket.list('%s/' % app_config.ASSETS_SLUG)

    which = None
    always = False

    for key in keys:
        download = False
        upload = False

        local_path = key.name.replace(app_config.ASSETS_SLUG, ASSETS_ROOT, 1)

        # Skip root key
        if local_path == '%s/' % ASSETS_ROOT:
            continue

        logger.info(local_path)

        if local_path in local_paths:
            # A file can only exist once, this speeds up future checks
            # and provides a list of non-existing files when complete
            local_paths.remove(local_path)

{'fraction_non_alphanumeric': 0.08269018743109151, 'fraction_numerical': 0.002205071664829107, 'mean_word_length': 2.2917271407837445, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '302687', 'n_tokens_mistral': 1372, 'n_tokens_neox': 1305, 'n_words': 377}