<commit_before>from osmread import parse_file, Way

from ott.utils import json_utils
from ott.utils import date_utils
from ott.utils import file_utils

import logging
log = logging.getLogger(__file__)



class OsmInfo(object):
    """ Utility for getting stats on an osm file 
    """
    def __init__(self):
        """ TODO think about rewriting this to simply read file line by line and find latest stamps and count tags...
            will be much faster than osmread on big files...
        """
        self.way_count = 0
        self.highway_count = 0
        class Last(object):
            timestamp = 0
            changeset = 0
        self.last = Last()

    def __str__(self):
        return json_utils.json_repr(self, pretty_print=True)

    def to_json(self):
        return json_utils.obj_to_dict(self)

    def write_json_file(self, file_path, pretty_print=False):
        json_utils.object_to_json_file(file_path, self, pretty_print)

    def calculate_osm_stats(self, osm_path):
        """ reads the .osm file and captures certain stats like last update and number of ways, etc... 
        """
        log.info("calculating stats for {}".format(osm_path))
        for entity in parse_file(osm_path):
            if isinstance(entity, Way):
                self.way_count += 1
                if 'highway' in entity.tags:
                    self.highway_count += 1
                if entity.timestamp > self.last.timestamp:
                    self.last.timestamp = entity.timestamp
                    self.last.changeset = entity.changeset

        # clean up data
        self.last.changeset_url = "http://openstreetmap.org/changeset/{}".format(self.last.changeset)
        self.last.edit_date = date_utils.pretty_date_from_ms(self.last.timestamp * 1000, fmt="%B %d, %Y")
        self.last.file_date = file_utils.file_pretty_date(osm_path, fmt="%B %d, %Y")

    @classmethod
    def find_osm_files(cls, dir_path):
        ret_val = []
        osm = file_utils.find_files(dir_path, ".osm")
        if osm:
            ret_val.extend(osm)
        pbf = file_utils.find_files(dir_path, ".pbf")
        if pbf:
            ret_val.extend(pbf)
        return ret_val

    @classmethod
    def get_stats_file_path(cls, osm_file, stats_file=None):
        if stats_file is None:
            stats_file = osm_file + "-stats"
        return stats_file

    @classmethod
    def cache_stats(cls, osm_path, stats_file=None, pretty_print=True):
        """ return OSM .osm-stats info json
            will either read the .osm-stats file or create a new stats file calculated via the .osm file
        """
        # import pdb; pdb.set_trace()
        stats_path = cls.get_stats_file_path(osm_path, stats_file)
        is_stats_good = False

        # step 1: read the .osm-stats cache file (if exists)
        is_stats_fresh = file_utils.is_a_newer_than_b(stats_path, osm_path)
        if is_stats_fresh:
            j = json_utils.get_json(stats_path)
            if j.get('way_count') and j.get('highway_count') and j.get('last'):
                is_stats_good = True
                ret_val = j

        # step 2: if .osm-stats don't exists or are out of date, cache them
        if not is_stats_good:
            stats = OsmInfo()
            stats.calculate_osm_stats(osm_path)
            stats_file = cls.get_stats_file_path(osm_path, stats_file)
            stats.write_json_file(stats_file, pretty_print)
            ret_val = stats.to_json()

        return ret_val
<commit_msg>redo logic of osm v stats<commit_after>from osmread import parse_file, Way

from ott.utils import json_utils
from ott.utils import date_utils
from ott.utils import file_utils

import logging
log = logging.getLogger(__file__)



class OsmInfo(object):
    """ Utility for getting stats on an osm file 
    """
    def __init__(self):
        """ TODO think about rewriting this to simply read file line by line and find latest stamps and count tags...
            will be much faster than osmread on big files...
        """
        self.way_count = 0
        self.highway_count = 0
        class Last(object):
            timestamp = 0
            changeset = 0
        self.last = Last()

    def __str__(self):
        return json_utils.json_repr(self, pretty_print=True)

    def to_json(self):
        return json_utils.obj_to_dict(self)

    def write_json_file(self, file_path, pretty_print=False):
        json_utils.object_to_json_file(file_path, self, pretty_print)

    def calculate_osm_stats(self, osm_path):
        """ reads the .osm file and captures certain stats like last update and number of ways, etc... 
        """
        log.info("calculating stats for {}".format(osm_path))
        for entity in parse_file(osm_path):
            if isinstance(entity, Way):
                self.way_count += 1
                if 'highway' in entity.tags:
                    self.highway_count += 1
                if entity.timestamp > self.last.timestamp:
                    self.last.timestamp = entity.timestamp
                    self.last.changeset = entity.changeset

        # clean up data
        self.last.changeset_url = "http://openstreetmap.org/changeset/{}".format(self.last.changeset)
        self.last.edit_date = date_utils.pretty_date_from_ms(self.last.timestamp * 1000, fmt="%B %d, %Y")
        self.last.file_date = file_utils.file_pretty_date(osm_path, fmt="%B %d, %Y")

    @classmethod
    def find_osm_files(cls, dir_path):
        ret_val = []
        osm = file_utils.find_files(dir_path, ".osm")
        if osm:
            ret_val.extend(osm)
        pbf = file_utils.find_files(dir_path, ".pbf")
        if pbf:
            ret_val.extend(pbf)
        return ret_val

    @classmethod
    def get_stats_file_path(cls, osm_file, stats_file=None):
        if stats_file is None:
            stats_file = osm_file + "-stats"
        return stats_file

    @classmethod
    def cache_stats(cls, osm_path, stats_file=None, pretty_print=True):
        """ return OSM .osm-stats info json
            will either read the .osm-stats file or create a new stats file calculated via the .osm file
        """
        # import pdb; pdb.set_trace()
        stats_path = cls.get_stats_file_path(osm_path, stats_file)
        is_stats_good = False

        # step 1: read the .osm-stats cache file (if exists and stats are newer than osm)
        if not file_utils.is_a_newer_than_b(osm_path, stats_path):
            j = json_utils.get_json(stats_path)
            if j.get('way_count') and j.get('highway_count') and j.get('last'):
                is_stats_good = True
                ret_val = j

        # step 2: if .osm-stats don't exists or are out of date, cache them
        if not is_stats_good:
            stats = OsmInfo()
            stats.calculate_osm_stats(osm_path)
            stats_file = cls.get_stats_file_path(osm_path, stats_file)
            stats.write_json_file(stats_file, pretty_print)
            ret_val = stats.to_json()

        return ret_val

{'fraction_non_alphanumeric': 0.08953305726117378, 'fraction_numerical': 0.003427102670284164, 'mean_word_length': 2.2637465051258157, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4481561', 'n_tokens_mistral': 2232, 'n_tokens_neox': 2134, 'n_words': 556}