<commit_before>from StringIO import StringIO
from io import FileIO
from os import path
import os
from uuid import uuid4
from collections import defaultdict
from copy import deepcopy
import shutil
import hashlib
import tempfile
from couchdbkit import ResourceConflict, ResourceNotFound
from casexml.apps.case.models import CommCareCase
from casexml.apps.phone.caselogic import BatchedCaseSyncOperation, CaseSyncUpdate
from casexml.apps.phone.exceptions import (
    MissingSyncLog, InvalidSyncLogException, SyncLogUserMismatch,
    BadStateException, RestoreException,
)
from casexml.apps.stock.consumption import compute_consumption_or_default
from casexml.apps.stock.utils import get_current_ledger_transactions_multi
from corehq.toggles import LOOSE_SYNC_TOKEN_VALIDATION, FILE_RESTORE, STREAM_RESTORE_CACHE, ENABLE_LOADTEST_USERS
from dimagi.utils.decorators.memoized import memoized
from dimagi.utils.parsing import json_format_datetime
from casexml.apps.phone.models import SyncLog
import logging
from dimagi.utils.couch.database import get_db, get_safe_write_kwargs
from casexml.apps.phone import xml
from datetime import datetime, timedelta
from casexml.apps.stock.const import COMMTRACK_REPORT_XMLNS
from dimagi.utils.couch.cache.cache_core import get_redis_default_cache
from couchforms.xml import (
    ResponseNature,
    get_simple_response_xml,
)
from casexml.apps.case.xml import check_version, V1
from casexml.apps.phone.fixtures import generator
from django.http import HttpResponse, StreamingHttpResponse
from django.conf import settings
from casexml.apps.phone.checksum import CaseStateHash
from no_exceptions.exceptions import HttpException
from wsgiref.util import FileWrapper

logger = logging.getLogger(__name__)

# how long a cached payload sits around for (in seconds).
INITIAL_SYNC_CACHE_TIMEOUT = 60 * 60  # 1 hour

# the threshold for setting a cached payload on initial sync (in seconds).
# restores that take less than this time will not be cached to allow
# for rapid iteration on fixtures/cases/etc.
INITIAL_SYNC_CACHE_THRESHOLD = 60  # 1 minute


def stream_response(payload, is_file=True):
    try:
        if is_file:
            response = StreamingHttpResponse(FileWrapper(open(payload, 'r')), mimetype="text/xml")
            response['Content-Length'] = os.path.getsize(payload)
            return response
        else:
            return StreamingHttpResponse(FileWrapper(payload), mimetype="text/xml")
    except IOError as e:
        return HttpResponse(e, status=500)
<commit_msg>unused import<commit_after>from StringIO import StringIO
from io import FileIO
from os import path
import os
from uuid import uuid4
from collections import defaultdict
from copy import deepcopy
import shutil
import hashlib
import tempfile
from couchdbkit import ResourceConflict, ResourceNotFound
from casexml.apps.case.models import CommCareCase
from casexml.apps.phone.caselogic import BatchedCaseSyncOperation, CaseSyncUpdate
from casexml.apps.phone.exceptions import (
    MissingSyncLog, InvalidSyncLogException, SyncLogUserMismatch,
    BadStateException, RestoreException,
)
from casexml.apps.stock.consumption import compute_consumption_or_default
from casexml.apps.stock.utils import get_current_ledger_transactions_multi
from corehq.toggles import LOOSE_SYNC_TOKEN_VALIDATION, FILE_RESTORE, STREAM_RESTORE_CACHE, ENABLE_LOADTEST_USERS
from dimagi.utils.decorators.memoized import memoized
from dimagi.utils.parsing import json_format_datetime
from casexml.apps.phone.models import SyncLog
import logging
from dimagi.utils.couch.database import get_db, get_safe_write_kwargs
from casexml.apps.phone import xml
from datetime import datetime, timedelta
from casexml.apps.stock.const import COMMTRACK_REPORT_XMLNS
from dimagi.utils.couch.cache.cache_core import get_redis_default_cache
from couchforms.xml import (
    ResponseNature,
    get_simple_response_xml,
)
from casexml.apps.case.xml import check_version, V1
from casexml.apps.phone.fixtures import generator
from django.http import HttpResponse, StreamingHttpResponse
from django.conf import settings
from casexml.apps.phone.checksum import CaseStateHash
from wsgiref.util import FileWrapper

logger = logging.getLogger(__name__)

# how long a cached payload sits around for (in seconds).
INITIAL_SYNC_CACHE_TIMEOUT = 60 * 60  # 1 hour

# the threshold for setting a cached payload on initial sync (in seconds).
# restores that take less than this time will not be cached to allow
# for rapid iteration on fixtures/cases/etc.
INITIAL_SYNC_CACHE_THRESHOLD = 60  # 1 minute


def stream_response(payload, is_file=True):
    try:
        if is_file:
            response = StreamingHttpResponse(FileWrapper(open(payload, 'r')), mimetype="text/xml")
            response['Content-Length'] = os.path.getsize(payload)
            return response
        else:
            return StreamingHttpResponse(FileWrapper(payload), mimetype="text/xml")
    except IOError as e:
        return HttpResponse(e, status=500)

{'fraction_non_alphanumeric': 0.05665665665665666, 'fraction_numerical': 0.005205205205205205, 'mean_word_length': 5.332065906210393, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 34}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6433446', 'n_tokens_mistral': 1446, 'n_tokens_neox': 1418, 'n_words': 429}