<commit_before>import argparse
from contextlib import contextmanager
import hashlib
import os
from random import Random
import shutil
import sys
import tempfile

import boto
from simplejson import dumps
from six.moves import range as xrange
from sqlalchemy import text

from ichnaea.config import read_config
from ichnaea.db import (
    configure_db,
    db_worker_session,
)
from ichnaea.log import (
    configure_raven,
    configure_stats,
)
from ichnaea import util

IMAGE_HEADERS = {
    'Content-Type': 'image/png',
    'Cache-Control': 'max-age=3600, public',
}
JSON_HEADERS = {
    'Content-Type': 'application/json',
    'Cache-Control': 'max-age=3600, public',
}


@contextmanager
def tempdir():
    workdir = tempfile.mkdtemp()
    try:
        yield workdir
    finally:
        shutil.rmtree(workdir)


def system_call(cmd):  # pragma: no cover
    # testing hook
    return os.system(cmd)


def export_to_csv(session, filename, multiplier=5):
    # Order by id to keep a stable ordering.
    stmt = text('select lat, lon from mapstat '
                'order by id limit :l offset :o')

    # Set up a pseudo random generator with a fixed seed to prevent
    # datamap tiles from changing with every generation.
    pseudorandom = Random()
    pseudorandom.seed(42)
    random = pseudorandom.random
    offset = 0
    batch = 200000
    pattern = '%.6f,%.6f\n'

    result_rows = 0
    # export mapstat mysql table as csv to local file
    with open(filename, 'w') as fd:
        while True:
            result = session.execute(stmt.bindparams(o=offset, l=batch))
            rows = result.fetchall()
            result.close()
            if not rows:
                break
            lines = []
            append = lines.append
            for row in rows:
                for i in xrange(multiplier):
                    # keep calling random even if we skip the lines
                    # to preserve the pseudo-random sequence
                    lat = (row[0] + random()) / 1000.0
                    lon = (row[1] + random()) / 1000.0
                    if row[0] != 0 or row[1] != 0:
                        append(pattern % (lat, lon))
            fd.writelines(lines)<commit_msg>Deal with mapstat lat/lon being set to null.<commit_after>import argparse
from contextlib import contextmanager
import hashlib
import os
from random import Random
import shutil
import sys
import tempfile

import boto
from simplejson import dumps
from six.moves import range as xrange
from sqlalchemy import text

from ichnaea.config import read_config
from ichnaea.db import (
    configure_db,
    db_worker_session,
)
from ichnaea.log import (
    configure_raven,
    configure_stats,
)
from ichnaea import util

IMAGE_HEADERS = {
    'Content-Type': 'image/png',
    'Cache-Control': 'max-age=3600, public',
}
JSON_HEADERS = {
    'Content-Type': 'application/json',
    'Cache-Control': 'max-age=3600, public',
}


@contextmanager
def tempdir():
    workdir = tempfile.mkdtemp()
    try:
        yield workdir
    finally:
        shutil.rmtree(workdir)


def system_call(cmd):  # pragma: no cover
    # testing hook
    return os.system(cmd)


def export_to_csv(session, filename, multiplier=5):
    # Order by id to keep a stable ordering.
    stmt = text('select lat, lon from mapstat '
                'order by id limit :l offset :o')

    # Set up a pseudo random generator with a fixed seed to prevent
    # datamap tiles from changing with every generation.
    pseudorandom = Random()
    pseudorandom.seed(42)
    random = pseudorandom.random
    offset = 0
    batch = 200000
    pattern = '%.6f,%.6f\n'

    result_rows = 0
    # export mapstat mysql table as csv to local file
    with open(filename, 'w') as fd:
        while True:
            result = session.execute(stmt.bindparams(o=offset, l=batch))
            rows = result.fetchall()
            result.close()
            if not rows:
                break
            lines = []
            append = lines.append
            for row in rows:
                for i in xrange(multiplier):
                    # keep calling random even if we skip the lines
                    # to preserve the pseudo-random sequence
                    if (row[0] is None or row[1] is None or
                            (row[0] == 0 and row[1] == 0)):
                        random()
                        random()
                    else:
                        lat = (row[0] + random()) / 1000.0
                        lon = (row[1] + random()) / 1000.0
                        append(pattern % (lat, lon))
            fd.writelines(lines)
{'fraction_non_alphanumeric': 0.0880952380952381, 'fraction_numerical': 0.01645021645021645, 'mean_word_length': 2.0622929092113984, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1512783', 'n_tokens_mistral': 1392, 'n_tokens_neox': 1311, 'n_words': 439}