<commit_before># Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import importlib_resources
import json

from functools import partial

import asks
import click
import trio

from linehaul.bigquery import BigQuery
from linehaul.migration import migrate as migrate_
from linehaul.server import server as server_


asks.init("trio")


@click.group(
    context_settings={
        "auto_envvar_prefix": "LINEHAUL",
        "help_option_names": ["-h", "--help"],
        "max_content_width": 88,
    }
)
def cli():
    """
    The Linehaul Statistics Daemon.

    Linehaul is a daemon that implements the syslog protocol, and listens for specially
    formatted messages corresponding to download events of Python packages. For each
    event it receives it processes them, and then loads them into a BigQuery database.
    """


@cli.command(short_help="Runs the Linehaul server.")
@click.option(
    "--credentials",
    type=click.File("r", encoding="utf8"),
    required=True,
    help="A path to the credentials JSON for a GCP service account.",
)
@click.option(
    "--bind", default="0.0.0.0", show_default=True, help="The IP address to bind to."
)
@click.option(
    "--port", type=int, default=512, show_default=True, help="The port to bind to."
)
@click.option("--token", help="A token used to authenticate a remote syslog stream.")
@click.option(
    "--queued-events",
    type=int,
    default=10000,
    show_default=True,
    help="How many events to queue for processing before applying backpressure.",
)
@click.option(
    "--batch-size",
    type=int,
    default=500,
    show_default=True,
    help="The number of events to send in each BigQuery API call.",
)
@click.option(
    "--batch-timeout",
    type=int,
    default=30,
    show_default=True,
    help=(
        "How long to wait before sending a smaller than --batch-size batch of events "
        "to BigQuery."
    ),
)
@click.option(
    "--api-timeout",
    type=int,
    default=30,
    show_default=True,
    help="How long to wait for a single API call to BigQuery to complete.",
)
@click.argument("table")
def server(
    credentials,
    bind,
    port,
    token,
    queued_events,
    batch_size,
    batch_timeout,
    api_timeout,
    table,
):
    """<commit_msg>Add metavar to make our help text better<commit_after># Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import importlib_resources
import json

from functools import partial

import asks
import click
import trio

from linehaul.bigquery import BigQuery
from linehaul.migration import migrate as migrate_
from linehaul.server import server as server_


asks.init("trio")


@click.group(
    context_settings={
        "auto_envvar_prefix": "LINEHAUL",
        "help_option_names": ["-h", "--help"],
        "max_content_width": 88,
    }
)
def cli():
    """
    The Linehaul Statistics Daemon.

    Linehaul is a daemon that implements the syslog protocol, and listens for specially
    formatted messages corresponding to download events of Python packages. For each
    event it receives it processes them, and then loads them into a BigQuery database.
    """


@cli.command(short_help="Runs the Linehaul server.")
@click.option(
    "--credentials",
    type=click.File("r", encoding="utf8"),
    required=True,
    help="A path to the credentials JSON for a GCP service account.",
)
@click.option(
    "--bind",
    default="0.0.0.0",
    metavar="ADDR",
    show_default=True,
    help="The IP address to bind to.",
)
@click.option(
    "--port",
    type=int,
    default=512,
    metavar="PORT",
    show_default=True,
    help="The port to bind to.",
)
@click.option("--token", help="A token used to authenticate a remote syslog stream.")
@click.option(
    "--queued-events",
    type=int,
    default=10000,
    show_default=True,
    help="How many events to queue for processing before applying backpressure.",
)
@click.option(
    "--batch-size",
    type=int,
    default=500,
    show_default=True,
    help="The number of events to send in each BigQuery API call.",
)
@click.option(
    "--batch-timeout",
    type=int,
    default=30,
    metavar="SECONDS",
    show_default=True,
    help=(
        "How long to wait before sending a smaller than --batch-size batch of events "
        "to BigQuery."
    ),
)
@click.option(
    "--api-timeout",
    type=int,
    default=30,
    metavar="SECONDS",
    show_default=True,
    help="How long to wait for a single API call to BigQuery to complete.",
)
@click.argument("table")
def server(
    credentials,
    bind,
    port,
    token,
    queued_events,
    batch_size,
    batch_timeout,
    api_timeout,
    table,
):
    """
{'fraction_non_alphanumeric': 0.10694617161129863, 'fraction_numerical': 0.009237875288683603, 'mean_word_length': 3.3508500772797527, 'pattern_counts': {'":': 6, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6489604', 'n_tokens_mistral': 1710, 'n_tokens_neox': 1651, 'n_words': 601}