<commit_before># encoding=utf-8

import collections
import itertools
import logging
import os, lzma
import shutil
import zipfile

from sqlalchemy import func
from sqlalchemy.sql.expression import delete

from terroroftinytown.format import registry
from terroroftinytown.format.projectsettings import ProjectSettingsWriter
from terroroftinytown.format.urlformat import quote
from terroroftinytown.tracker.bootstrap import Bootstrap
from terroroftinytown.tracker.model import new_session, Project, Result
from terroroftinytown.util.externalsort import GNUExternalSort


logger = logging.getLogger(__name__)


ResultContainer = collections.namedtuple(
    'ResultContainer',
    ['id', 'shortcode', 'url', 'encoding', 'datetime']
)


class Exporter:
    def __init__(self, output_dir, format="beacon", settings={}):
        super().__init__()

        self.setup_format(format)
        self.output_dir = output_dir
        self.settings = settings
        self.after = self.settings['after']
        self.max_items = self.settings['max_items']

        self.projects_count = 0
        self.items_count = 0
        self.last_date = None

        self.lzma = True
        self.extension = 'txt.xz'

        # Length of directory name
        self.dir_length = settings['dir_length']
        # Number of characters from the right are not used in directory name
        # in other words, number of _
        self.max_right = settings['max_right']
        # Number of characters from the left that are used in file name
        # in other words, number of characters that are not in directory name and not _
        self.file_length = settings['file_length']

        # Example of settings:
        # dir_length = 2
        # max_right = 4
        # file_length = 2
        # output: projectname/00/01/000100____.txt, projectname/01/01__.txt

        self.fp = None
        self.writer = None
        self.project_result_sorters = {}

    def setup_format(self, format):
        self.format = registry[format]

    def make_output_dir(self):
        if not os.path.isdir(self.output_dir):
            os.makedirs(self.output_dir)

    def dump(self):
        self.make_output_dir()

        with new_session() as session:
            self._input_sorters(session)

            for project_id, sorter in self.project_result_sorters.items():
                project = session.query(Project).filter_by(name=project_id).first()

                if self.settings['include_settings']:
                    self.dump_project_settings(project)

                self.dump_project(project, sorter, session)

                if self.settings['zip']:
                    self.zip_project(project)

    def _input_sorters(self, session, size=1000):
        query = session.query(Result)
        if self.after:
            query = query.filter(Result.datetime > self.after)

        for offset in itertools.count(step=size):
            # Optimized for SQLite scrolling window
            rows = query.filter(Result.id >= offset).limit(size).all()

            if not rows:
                break

            for result in rows:
                if result.project_id not in self.project_result_sorters:
                    self.project_result_sorters[result.project_id] = \
                        GNUExternalSort(temp_dir=self.output_dir,
                                        temp_prefix='tott-{0}-'.format(
                                            result.project_id
                                            )
                                        )
                    self.projects_count += 1

                sorter = self.project_result_sorters[result.project_id]
                sorter.input(
                    result.shortcode,
                    (result.id, result.url, result.encoding, result.datetime)
                )
                self.items_count += 1

                if self.items_count % 10000 == 0:<commit_msg>tracker.export: Fix scroll window to use last row id.<commit_after># encoding=utf-8

import collections
import itertools
import logging
import os, lzma
import shutil
import zipfile

from sqlalchemy import func
from sqlalchemy.sql.expression import delete

from terroroftinytown.format import registry
from terroroftinytown.format.projectsettings import ProjectSettingsWriter
from terroroftinytown.format.urlformat import quote
from terroroftinytown.tracker.bootstrap import Bootstrap
from terroroftinytown.tracker.model import new_session, Project, Result
from terroroftinytown.util.externalsort import GNUExternalSort


logger = logging.getLogger(__name__)


ResultContainer = collections.namedtuple(
    'ResultContainer',
    ['id', 'shortcode', 'url', 'encoding', 'datetime']
)


class Exporter:
    def __init__(self, output_dir, format="beacon", settings={}):
        super().__init__()

        self.setup_format(format)
        self.output_dir = output_dir
        self.settings = settings
        self.after = self.settings['after']
        self.max_items = self.settings['max_items']

        self.projects_count = 0
        self.items_count = 0
        self.last_date = None

        self.lzma = True
        self.extension = 'txt.xz'

        # Length of directory name
        self.dir_length = settings['dir_length']
        # Number of characters from the right are not used in directory name
        # in other words, number of _
        self.max_right = settings['max_right']
        # Number of characters from the left that are used in file name
        # in other words, number of characters that are not in directory name and not _
        self.file_length = settings['file_length']

        # Example of settings:
        # dir_length = 2
        # max_right = 4
        # file_length = 2
        # output: projectname/00/01/000100____.txt, projectname/01/01__.txt

        self.fp = None
        self.writer = None
        self.project_result_sorters = {}

    def setup_format(self, format):
        self.format = registry[format]

    def make_output_dir(self):
        if not os.path.isdir(self.output_dir):
            os.makedirs(self.output_dir)

    def dump(self):
        self.make_output_dir()

        with new_session() as session:
            self._input_sorters(session)

            for project_id, sorter in self.project_result_sorters.items():
                project = session.query(Project).filter_by(name=project_id).first()

                if self.settings['include_settings']:
                    self.dump_project_settings(project)

                self.dump_project(project, sorter, session)

                if self.settings['zip']:
                    self.zip_project(project)

    def _input_sorters(self, session, size=1000):
        query = session.query(Result)
        if self.after:
            query = query.filter(Result.datetime > self.after)

        last_id = -1

        while True:
            # Optimized for SQLite scrolling window
            rows = query.filter(Result.id > last_id).limit(size).all()

            if not rows:
                break

            for result in rows:
                if result.project_id not in self.project_result_sorters:
                    self.project_result_sorters[result.project_id] = \
                        GNUExternalSort(temp_dir=self.output_dir,
                                        temp_prefix='tott-{0}-'.format(
                                            result.project_id
                                            )
                                        )
                    self.projects_count += 1

                sorter = self.project_result_sorters[result.project_id]
                sorter.input(
                    result.shortcode,
                    (result.id, result.url, result.encoding, result.datetime)
                )
                self.items_count += 1
                last_id = result.id

                if self.items_count % 10000 == 0:
{'fraction_non_alphanumeric': 0.08118361153262518, 'fraction_numerical': 0.008472432979261507, 'mean_word_length': 1.9879108424631657, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1193063', 'n_tokens_mistral': 2167, 'n_tokens_neox': 2035, 'n_words': 503}