<commit_before>import numpy as np
import os
import os.path
import shutil

from opensfm import config
from opensfm import io
from opensfm.dataset import DataSet


class MetaDataSet():
    def __init__(self, data_path):
        '''
        Create meta dataset instance for large scale reconstruction.

        :param data_path: Path to directory containing meta dataset
        '''
        self.data_path = os.path.abspath(data_path)

        config_file = os.path.join(self.data_path, 'config.yaml')
        self.config = config.load_config(config_file)

        self._image_list_file_name = 'image_list_with_gps.tsv'
        self._clusters_file_name = 'clusters.npz'
        self._clusters_with_neighbors_file_name = 'clusters_with_neighbors.npz'
        self._clusters_with_neighbors_geojson_file_name = 'clusters_with_neighbors.geojson'
        self._clusters_geojson_file_name = 'clusters.geojson'

        io.mkdir_p(self._submodels_path())

    def _submodels_path(self):
        return os.path.join(self.data_path, self.config['submodels_relpath'])

    def _submodel_path(self, i):
        """Path to submodel i folder."""
        template = self.config['submodel_relpath_template']
        return os.path.join(self.data_path, template % i)

    def _submodel_images_path(self, i):
        """Path to submodel i images folder."""
        template = self.config['submodel_images_relpath_template']
        return os.path.join(self.data_path, template % i)

    def _image_groups_path(self):
        return os.path.join(self.data_path, 'image_groups.txt')

    def _image_list_path(self):
        return os.path.join(self._submodels_path(), self._image_list_file_name)

    def _clusters_path(self):
        return os.path.join(self._submodels_path(), self._clusters_file_name)

    def _clusters_with_neighbors_path(self):
        return os.path.join(self._submodels_path(), self._clusters_with_neighbors_file_name)

    def _clusters_with_neighbors_geojson_path(self):
        return os.path.join(self._submodels_path(), self._clusters_with_neighbors_geojson_file_name)

    def _clusters_geojson_path(self):
        return os.path.join(self._submodels_path(), self._clusters_geojson_file_name)

    def _create_symlink(self, base_path, dir_name):
        link_path = os.path.join(base_path, dir_name)

        if os.path.islink(link_path):
            os.unlink(link_path)

        os.symlink(
            os.path.relpath(os.path.join(self.data_path, dir_name), base_path),
            os.path.join(link_path))

    def image_groups_exists(self):
        return os.path.isfile(self._image_groups_path())

    def load_image_groups(self):
        with open(self._image_groups_path()) as fin:
            for line in fin:
                yield line.split()

    def image_list_exists(self):<commit_msg>feat(split/merge): symlink only existing files<commit_after>import numpy as np
import os
import os.path
import shutil

from opensfm import config
from opensfm import io
from opensfm.dataset import DataSet


class MetaDataSet():
    def __init__(self, data_path):
        '''
        Create meta dataset instance for large scale reconstruction.

        :param data_path: Path to directory containing meta dataset
        '''
        self.data_path = os.path.abspath(data_path)

        config_file = os.path.join(self.data_path, 'config.yaml')
        self.config = config.load_config(config_file)

        self._image_list_file_name = 'image_list_with_gps.tsv'
        self._clusters_file_name = 'clusters.npz'
        self._clusters_with_neighbors_file_name = 'clusters_with_neighbors.npz'
        self._clusters_with_neighbors_geojson_file_name = 'clusters_with_neighbors.geojson'
        self._clusters_geojson_file_name = 'clusters.geojson'

        io.mkdir_p(self._submodels_path())

    def _submodels_path(self):
        return os.path.join(self.data_path, self.config['submodels_relpath'])

    def _submodel_path(self, i):
        """Path to submodel i folder."""
        template = self.config['submodel_relpath_template']
        return os.path.join(self.data_path, template % i)

    def _submodel_images_path(self, i):
        """Path to submodel i images folder."""
        template = self.config['submodel_images_relpath_template']
        return os.path.join(self.data_path, template % i)

    def _image_groups_path(self):
        return os.path.join(self.data_path, 'image_groups.txt')

    def _image_list_path(self):
        return os.path.join(self._submodels_path(), self._image_list_file_name)

    def _clusters_path(self):
        return os.path.join(self._submodels_path(), self._clusters_file_name)

    def _clusters_with_neighbors_path(self):
        return os.path.join(self._submodels_path(), self._clusters_with_neighbors_file_name)

    def _clusters_with_neighbors_geojson_path(self):
        return os.path.join(self._submodels_path(), self._clusters_with_neighbors_geojson_file_name)

    def _clusters_geojson_path(self):
        return os.path.join(self._submodels_path(), self._clusters_geojson_file_name)

    def _create_symlink(self, base_path, filename):
        src = os.path.join(self.data_path, filename)
        dst = os.path.join(base_path, filename)

        if not os.path.exists(src):
            return

        if os.path.islink(dst):
            os.unlink(dst)

        os.symlink(os.path.relpath(src, base_path), dst)

    def image_groups_exists(self):
        return os.path.isfile(self._image_groups_path())

    def load_image_groups(self):
        with open(self._image_groups_path()) as fin:
            for line in fin:
                yield line.split()

    def image_list_exists(self):
{'fraction_non_alphanumeric': 0.09681978798586573, 'fraction_numerical': 0.0, 'mean_word_length': 3.351268255188317, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7459467', 'n_tokens_mistral': 1865, 'n_tokens_neox': 1806, 'n_words': 324}