<commit_before>
import logging

import requests
import pandas

from natsort import natsorted

LOG = logging.getLogger()

class Netdata:
    def __init__(self, config):
        self.url = f"{config['url']}/api/v1/data"
        self.metrics = config['metrics']
        self.charts = config['charts']

    def _get_data(self, timerange, resolution):
        data = pandas.DataFrame()
        for chart, dimensions in self.charts.items():
            response = requests.get(self.url, params={
                'chart': chart,
                'after': timerange[0],
                'before': timerange[1],
                'dimensions': ','.join(dimensions),
                'gtime': resolution
            })

            status_code = response.status_code
            if status_code != 200:
                LOG.warning(f'Netdata response not 200, but {status_code} '
                            f'for chart {chart}.')
                continue

            result = response.json()

            columns = ['time']
            columns.extend([f'{chart}.{dimension}' for dimension in dimensions])
            columns = [column.replace('.', '_') for column in columns]

            df = pandas.DataFrame(result['data'], columns=columns)
            df = df.set_index('time')
            data = pandas.concat([data, df], axis=1)

        data.index = pandas.to_datetime(data.index, unit='s')

        return data

    @classmethod
    def make_timestamp(cls, value):
        return int(value.timestamp())

    def _get_netdata_per_query(self, df, output):
        data = {}

        for _, row in df.iterrows():
            timerange = (
                Netdata.make_timestamp(row['timestamp_start']),
                Netdata.make_timestamp(row['timestamp_stop'])
            )

            netdata_df = self._get_data(timerange, 1)

            query_id = row['query_id']
            data[query_id] = netdata_df.agg(self.metrics)

        return data

    def get_system_stats(self, df, resolution):
        ts_from = Netdata.make_timestamp(df['timestamp_start'].min())
        ts_to = Netdata.make_timestamp(df['timestamp_stop'].max())
        return self._get_data((ts_from, ts_to), resolution).sort_index()

    def _write_stats_per_query(self, df, output):
        data = self._get_netdata_per_query(df, output)
        query_ids = natsorted(data.keys())

        with open(output, 'w') as output_file:
            for query_id in query_ids:
                output_file.write(f'{query_id}')
                data[query_id].to_csv(output_file)
                output_file.write('\n')

    def _write_stats_no_breakdown(self, df, output):
        LOG.info('Running more than one stream. Netdata stats are written '
                 'out without analysis.')
        netdata_df = self.get_system_stats(df, 1)

        with open(output, 'w') as output_file:
            output_file.write(f'all')
            netdata_df.to_csv(output_file)
            output_file.write('\n')

    def write_stats(self, df, output):
        if len(df['stream_id'].unique()) == 1:
            self._write_stats_per_query(df, output)
        else:
            self._write_stats_no_breakdown(df, output)<commit_msg>Bugfix/newline in netdata output (#56)<commit_after>import os
import logging

import requests
import pandas

from natsort import natsorted

LOG = logging.getLogger()

class Netdata:
    def __init__(self, config):
        self.url = f"{config['url']}/api/v1/data"
        self.metrics = config['metrics']
        self.charts = config['charts']

    def _get_data(self, timerange, resolution):
        data = pandas.DataFrame()
        for chart, dimensions in self.charts.items():
            response = requests.get(self.url, params={
                'chart': chart,
                'after': timerange[0],
                'before': timerange[1],
                'dimensions': ','.join(dimensions),
                'gtime': resolution
            })

            status_code = response.status_code
            if status_code != 200:
                LOG.warning(f'Netdata response not 200, but {status_code} '
                            f'for chart {chart}.')
                continue

            result = response.json()

            columns = ['time']
            columns.extend([f'{chart}.{dimension}' for dimension in dimensions])
            columns = [column.replace('.', '_') for column in columns]

            df = pandas.DataFrame(result['data'], columns=columns)
            df = df.set_index('time')
            data = pandas.concat([data, df], axis=1)

        data.index = pandas.to_datetime(data.index, unit='s')

        return data

    @classmethod
    def make_timestamp(cls, value):
        return int(value.timestamp())

    def _get_netdata_per_query(self, df, output):
        data = {}

        for _, row in df.iterrows():
            timerange = (
                Netdata.make_timestamp(row['timestamp_start']),
                Netdata.make_timestamp(row['timestamp_stop'])
            )

            netdata_df = self._get_data(timerange, 1)

            query_id = row['query_id']
            data[query_id] = netdata_df.agg(self.metrics)

        return data

    def get_system_stats(self, df, resolution):
        ts_from = Netdata.make_timestamp(df['timestamp_start'].min())
        ts_to = Netdata.make_timestamp(df['timestamp_stop'].max())
        return self._get_data((ts_from, ts_to), resolution).sort_index()

    def _write_stats_per_query(self, df, output):
        data = self._get_netdata_per_query(df, output)
        query_ids = natsorted(data.keys())

        with open(output, 'w') as output_file:
            for query_id in query_ids:
                output_file.write(f'{query_id}\n')
                data[query_id].to_csv(output_file, header=False)
                output_file.write('\n')

    def _write_stats_no_breakdown(self, df, output):
        LOG.info('Running more than one stream. Netdata stats are written '
                 'out without analysis.')
        netdata_df = self.get_system_stats(df, 1)

        with open(output, 'w') as output_file:
            netdata_df.to_csv(output_file)
            output_file.write('\n')

    def write_stats(self, df, output):

        if len(df['stream_id'].unique()) == 1:
            flname, flext = os.path.splitext(output)
            self._write_stats_per_query(df, f'{flname}_single_stream{flext}')

        self._write_stats_no_breakdown(df, output)
{'fraction_non_alphanumeric': 0.11904392363805681, 'fraction_numerical': 0.004345801645196337, 'mean_word_length': 2.165029469548134, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2995908', 'n_tokens_mistral': 1949, 'n_tokens_neox': 1848, 'n_words': 388}