<commit_before># -*- coding: utf-8 -*-

import os
import json
import logging
import pickle
import gzip

import cv2
import numpy as np
import networkx as nx
import six

from opensfm import io
from opensfm import config
from opensfm import context


logger = logging.getLogger(__name__)


class DataSet:
    """Accessors to the main input and output data.

    Data include input images, masks, and segmentation as well
    temporary data such as feature and matches and the final
    reconstructions.

    All data is stored inside a single folder with a specific subfolder
    structure.

    It is possible to store data remotely or in different formats
    by subclassing this class and overloading its methods.
    """
    def __init__(self, data_path):
        """Init dataset associated to a folder."""
        self.data_path = data_path
        self._load_config()
        self._load_image_list()
        self._load_mask_list()

    def _load_config(self):
        config_file = os.path.join(self.data_path, 'config.yaml')
        self.config = config.load_config(config_file)

    def _load_image_list(self):
        """Load image list from image_list.txt or list image/ folder."""<commit_msg>chore: use new-style class<commit_after># -*- coding: utf-8 -*-

import os
import json
import logging
import pickle
import gzip

import cv2
import numpy as np
import networkx as nx
import six

from opensfm import io
from opensfm import config
from opensfm import context


logger = logging.getLogger(__name__)


class DataSet(object):
    """Accessors to the main input and output data.

    Data include input images, masks, and segmentation as well
    temporary data such as feature and matches and the final
    reconstructions.

    All data is stored inside a single folder with a specific subfolder
    structure.

    It is possible to store data remotely or in different formats
    by subclassing this class and overloading its methods.
    """
    def __init__(self, data_path):
        """Init dataset associated to a folder."""
        self.data_path = data_path
        self._load_config()
        self._load_image_list()
        self._load_mask_list()

    def _load_config(self):
        config_file = os.path.join(self.data_path, 'config.yaml')
        self.config = config.load_config(config_file)

    def _load_image_list(self):
        """Load image list from image_list.txt or list image/ folder."""
{'fraction_non_alphanumeric': 0.06856670797191243, 'fraction_numerical': 0.0016522098306484924, 'mean_word_length': 3.1830742659758204, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '67235', 'n_tokens_mistral': 674, 'n_tokens_neox': 655, 'n_words': 268}