<commit_before>import json
import logging
import os
import re
import subprocess
from collections import defaultdict
from shutil import copyfile
from subprocess import call

from django.conf import settings

import yaml

from corehq.apps.hqwebapp.exceptions import ResourceVersionsNotFoundException
from corehq.apps.hqwebapp.management.commands.resource_static import \
    Command as ResourceStaticCommand

logger = logging.getLogger('__name__')
ROOT_DIR = settings.FILEPATH
BUILD_JS_FILENAME = "staticfiles/build.js"
BUILD_TXT_FILENAME = "staticfiles/build.txt"


class Command(ResourceStaticCommand):
    help = '''
        Runs RequireJS optimizer to concatenate, minify, and bundle JavaScript files
        and set them up with the CDN.
    '''



    def add_arguments(self, parser):
        parser.add_argument('--local', action='store_true',
            help='Running on a local environment. Copies generated files back into corehq. Allows you to mimic '
                 'production optimization (dependency tracing, concatenation and minification) locally. '
                 'Does not allow you to mimic CDN.')
        parser.add_argument('--no_optimize', action='store_true',
            help='Don\'t minify files. Runs much faster. Useful when running on a local environment.')

    def handle(self, **options):
        logger.setLevel('DEBUG')

        local = options['local']
        no_optimize = options['no_optimize']

        if local:
            _confirm_or_exit()

        # During deploy, resource_static should already have run and populated resource_versions
        from get_resource_versions import get_resource_versions
        resource_versions = get_resource_versions()
        if (not resource_versions):
            raise ResourceVersionsNotFoundException()

        config, local_js_dirs = _r_js(local=local, no_optimize=no_optimize)

        if local:
            _copy_modules_back_into_corehq(config, local_js_dirs)

        filename = os.path.join(ROOT_DIR, 'staticfiles', 'hqwebapp', 'js', 'requirejs_config.js')
        resource_versions["hqwebapp/js/requirejs_config.js"] = self.get_hash(filename)
        if local:
            dest = os.path.join(ROOT_DIR, 'corehq', 'apps', 'hqwebapp', 'static',
                                'hqwebapp', 'js', 'requirejs_config.js')
            copyfile(filename, dest)
            logger.info("Copied updated requirejs_config.js back into {}".format(_relative(dest)))

        # Overwrite each bundle in resource_versions with the sha from the optimized version in staticfiles
        for module in config['modules']:
            filename = os.path.join(ROOT_DIR, 'staticfiles', module['name'] + ".js")

            # TODO: it'd be a performance improvement to do this after the `open` below
            # and pass in the file contents, since get_hash does another read.
            file_hash = self.get_hash(filename)

            # Overwrite source map reference. Source maps are accessed on the CDN,
            # so they need to have the version hash appended.
            with open(filename, 'r') as fin:
                lines = fin.readlines()
            with open(filename, 'w') as fout:
                for line in lines:
                    if re.search(r'sourceMappingURL=bundle.js.map$', line):
                        line = re.sub(r'bundle.js.map', 'bundle.js.map?version=' + file_hash, line)
                    fout.write(line)
            resource_versions[module['name'] + ".js"] = file_hash

        # Write out resource_versions.js for all js files in resource_versions
        # Exclude formdesigner directory, which contains a ton of files, none of which are required by HQ
        filename = os.path.join(ROOT_DIR, 'staticfiles', 'hqwebapp', 'js', 'resource_versions.js')
        with open(filename, 'w') as fout:
            fout.write("requirejs.config({ paths: %s });" % json.dumps({
                file[:-3]: "{}{}{}{}".format(settings.STATIC_CDN, settings.STATIC_URL, file[:-3],
                                             ".js?version=%s" % version if version else "")
                for file, version in resource_versions.items()
                if file.endswith(".js") and not file.startswith("formdesigner")
            }, indent=2))<commit_msg>Added new cache buster to users/js/bundle on each deploy<commit_after>import json
import logging
import os
import re
import subprocess
from collections import defaultdict
from uuid import uuid4
from shutil import copyfile
from subprocess import call

from django.conf import settings

import yaml

from corehq.apps.hqwebapp.exceptions import ResourceVersionsNotFoundException
from corehq.apps.hqwebapp.management.commands.resource_static import \
    Command as ResourceStaticCommand

logger = logging.getLogger('__name__')
ROOT_DIR = settings.FILEPATH
BUILD_JS_FILENAME = "staticfiles/build.js"
BUILD_TXT_FILENAME = "staticfiles/build.txt"


class Command(ResourceStaticCommand):
    help = '''
        Runs RequireJS optimizer to concatenate, minify, and bundle JavaScript files
        and set them up with the CDN.
    '''



    def add_arguments(self, parser):
        parser.add_argument('--local', action='store_true',
            help='Running on a local environment. Copies generated files back into corehq. Allows you to mimic '
                 'production optimization (dependency tracing, concatenation and minification) locally. '
                 'Does not allow you to mimic CDN.')
        parser.add_argument('--no_optimize', action='store_true',
            help='Don\'t minify files. Runs much faster. Useful when running on a local environment.')

    def handle(self, **options):
        logger.setLevel('DEBUG')

        local = options['local']
        no_optimize = options['no_optimize']

        if local:
            _confirm_or_exit()

        # During deploy, resource_static should already have run and populated resource_versions
        from get_resource_versions import get_resource_versions
        resource_versions = get_resource_versions()
        if (not resource_versions):
            raise ResourceVersionsNotFoundException()

        config, local_js_dirs = _r_js(local=local, no_optimize=no_optimize)

        if local:
            _copy_modules_back_into_corehq(config, local_js_dirs)

        filename = os.path.join(ROOT_DIR, 'staticfiles', 'hqwebapp', 'js', 'requirejs_config.js')
        resource_versions["hqwebapp/js/requirejs_config.js"] = self.get_hash(filename)
        if local:
            dest = os.path.join(ROOT_DIR, 'corehq', 'apps', 'hqwebapp', 'static',
                                'hqwebapp', 'js', 'requirejs_config.js')
            copyfile(filename, dest)
            logger.info("Copied updated requirejs_config.js back into {}".format(_relative(dest)))

        # Overwrite each bundle in resource_versions with the sha from the optimized version in staticfiles
        for module in config['modules']:
            filename = os.path.join(ROOT_DIR, 'staticfiles', module['name'] + ".js")

            # TODO: it'd be a performance improvement to do this after the `open` below
            # and pass in the file contents, since get_hash does another read.
            file_hash = self.get_hash(filename)

            if module['name'] == "users/js/bundle":
                file_hash = uuid4().hex

            # Overwrite source map reference. Source maps are accessed on the CDN,
            # so they need to have the version hash appended.
            with open(filename, 'r') as fin:
                lines = fin.readlines()
            with open(filename, 'w') as fout:
                for line in lines:
                    if re.search(r'sourceMappingURL=bundle.js.map$', line):
                        line = re.sub(r'bundle.js.map', 'bundle.js.map?version=' + file_hash, line)
                    fout.write(line)
            resource_versions[module['name'] + ".js"] = file_hash

        # Write out resource_versions.js for all js files in resource_versions
        # Exclude formdesigner directory, which contains a ton of files, none of which are required by HQ
        filename = os.path.join(ROOT_DIR, 'staticfiles', 'hqwebapp', 'js', 'resource_versions.js')
        with open(filename, 'w') as fout:
            fout.write("requirejs.config({ paths: %s });" % json.dumps({
                file[:-3]: "{}{}{}{}".format(settings.STATIC_CDN, settings.STATIC_URL, file[:-3],
                                             ".js?version=%s" % version if version else "")
                for file, version in resource_versions.items()
                if file.endswith(".js") and not file.startswith("formdesigner")
            }, indent=2))
{'fraction_non_alphanumeric': 0.09692289961968423, 'fraction_numerical': 0.0009219776420421805, 'mean_word_length': 2.6401006711409396, 'pattern_counts': {'":': 1, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1215714', 'n_tokens_mistral': 2399, 'n_tokens_neox': 2317, 'n_words': 696}