<commit_before>'''
Merges all the monthly files for a given sensor into
a complete time series of gas flaring.

There are two different approaches, a monthly one and annual one.

Algorithm is as follows:
1. Set the current month
2. Read in 12 months worth of monthly data from the current month appending the year and month and counter to each.
3. Add useful reduction paramters to the dataframe
4. Reduce by lats and lons and count the number of times each lat lon combination appears
5. Retain only those flares appear more than minimum sampling
6. Save that dataframe as the annual flaring characteristics during the 12 months from the current month
7. Inner join the annual and monthly dataframes on the lats and lons, keeping only those flares in the current
 month that were observed at least 4 times over the year.
8. Deal with last twelve months



'''

import logging
import glob
import os
import re

import numpy as np
import pandas as pd

import src.config.filepaths as fp


def main():
    # define sensor
    sensor = 'ats'

    # get all csv files for sensor
    root = os.path.join(fp.path_to_test_csv_out, sensor)
    csv_filepaths = glob.glob(root + '/*/*.csv')

    # filter the csv filepaths to just get the raw month files (as sometimes we might have other outputs in here)
    csv_filepaths = [f for f in csv_filepaths if re.search(r'[0-9]+.csv', f)]

    print csv_filepaths
    return<commit_msg>changes form work mac<commit_after>'''
Merges all the monthly files for a given sensor into
a complete time series of gas flaring.

There are two different approaches, a monthly one and annual one.

Algorithm is as follows:
1. Set the current month
2. Read in 12 months worth of monthly data from the current month appending the year and month and counter to each.
3. Add useful reduction paramters to the dataframe
4. Reduce by lats and lons and count the number of times each lat lon combination appears
5. Retain only those flares appear more than minimum sampling
6. Save that dataframe as the annual flaring characteristics during the 12 months from the current month
7. Inner join the annual and monthly dataframes on the lats and lons, keeping only those flares in the current
 month that were observed at least 4 times over the year.
8. Deal with last twelve months



'''

import logging
import glob
import os
import re

import numpy as np
import pandas as pd

import src.config.filepaths as fp


def main():
    # define sensor
    sensor = 'ats'

    # get all csv files for sensor
    root = os.path.join(fp.path_to_test_csv_out, sensor)
    csv_filepaths = glob.glob(root + '/*/*.csv')

    # filter the csv filepaths to just get the raw month files (as sometimes we might have other outputs in here)
    csv_filepaths = [f for f in csv_filepaths if re.search(r'[0-9]+.csv', f)]
{'fraction_non_alphanumeric': 0.0498220640569395, 'fraction_numerical': 0.010676156583629894, 'mean_word_length': 3.7323232323232323, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1196140', 'n_tokens_mistral': 786, 'n_tokens_neox': 749, 'n_words': 437}