<commit_before># Copyright 2018 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import sys
import tempfile

import click
import pkg_resources

import docuploader.credentials
import docuploader.log
import docuploader.tar
import docuploader.upload

try:
    VERSION = pkg_resources.get_distribution("gcp-docuploader").version
except pkg_resources.DistributionNotFound:
    VERSION = "0.0.0+dev"


@click.command()
@click.version_option(message="%(version)s", version=VERSION)
@click.option(
    "--credentials",
    default=docuploader.credentials.find(),
    help="Path to the credentials file to use for Google Cloud Storage.",
)
@click.argument("documentation_path")
# TODO: Metadata
def main(credentials, documentation_path):
    if not credentials:
        docuploader.log.error(
            "You need credentials to run this! Specify --credentials on the command line."
        )
        return sys.exit(1)

    docuploader.log.info(
        f"Sit tight, I'm tarring up your docs in {documentation_path}."
    )

    with tempfile.NamedTemporaryFile() as fh:
        fh.close()
        tar_filename = fh.name

        docuploader.tar.compress(documentation_path, tar_filename)

        docuploader.log.success(f"Cool, I have those tarred up.")

        docuploader.log.info(f"Okay, I'm sending them to the cloud™ now.")

        docuploader.upload.upload(
            source=tar_filename,
            # TODO: Destination filename should be based on metadata.
            destination="test.tar.gz",
            # TODO: Bucket should be an argument / well-known constant, I guess?
            bucket="docs-resources",
            credentials_file=credentials,
        )

    docuploader.log.success(f"All is well, your docs were uploaded! <3")


if __name__ == "__main__":
    main()<commit_msg>Add command to create metadata and use metadata during upload<commit_after># Copyright 2018 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import datetime
import pathlib
import shutil
import sys
import tempfile
from typing import Optional

import click
from google.protobuf import text_format  # type: ignore
import pkg_resources

from docuploader.protos import metadata_pb2
import docuploader.credentials
import docuploader.log
import docuploader.tar
import docuploader.upload

try:
    VERSION = pkg_resources.get_distribution("gcp-docuploader").version
except pkg_resources.DistributionNotFound:
    VERSION = "0.0.0+dev"

DEFAULT_STAGING_BUCKET = "docs-resources"


@click.group()
@click.version_option(message="%(version)s", version=VERSION)
def main():
    pass


@main.command()
@click.option(
    "--staging-bucket",
    default=DEFAULT_STAGING_BUCKET,
    help="The bucket to upload the staged documentation to.",
)
@click.option(
    "--credentials",
    default=docuploader.credentials.find(),
    help="Path to the credentials file to use for Google Cloud Storage.",
)
@click.option("--metadata-file", default=None, help="Path to the docs.metadata file.")
@click.argument("documentation_path")
def upload(
    staging_bucket: str,
    credentials: str,
    metadata_file: Optional[str],
    documentation_path: str,
):
    if not credentials:
        docuploader.log.error(
            "You need credentials to run this! Specify --credentials on the command line."
        )
        return sys.exit(1)

    if metadata_file is None:
        metadata_path = pathlib.Path(documentation_path) / "docs.metadata"
    else:
        metadata_path = pathlib.Path(metadata_file)

    if not metadata_path.exists():
        docuploader.log.error(
            "You need metadata to upload the docs. You can generate it with docuploader create-metadata"
        )
        return sys.exit(1)

    docuploader.log.success("Let's upload some docs!")

    docuploader.log.info("Loading up your metadata.")
    try:
        shutil.copy(metadata_path, pathlib.Path(documentation_path) / "docs.metadata")
    except shutil.SameFileError:
        pass

    metadata = metadata_pb2.Metadata()
    text_format.Merge(metadata_path.read_text(), metadata)
    # TODO: Validate that the minimum metadata is set.

    docuploader.log.success(
        f"Looks like we're uploading {metadata.name} version {metadata.version} for {metadata.language}."
    )

    docuploader.log.info(
        f"Sit tight, I'm tarring up your docs in {documentation_path}."
    )

    with tempfile.NamedTemporaryFile() as fh:
        fh.close()
        tar_filename = fh.name

        docuploader.tar.compress(documentation_path, tar_filename)

        docuploader.log.success(f"Cool, I have those tarred up.")

        docuploader.log.info(f"Okay, I'm sending them to the cloud™ now.")

        destination_name = (
            f"{metadata.language}-{metadata.name}-{metadata.version}.tar.gz"
        )

        docuploader.upload.upload(
            source=tar_filename,
            destination=destination_name,
            bucket=staging_bucket,
            credentials_file=credentials,
        )

    docuploader.log.success(
        f"All is well, your docs were uploaded to gs://{staging_bucket}/{destination_name}! <3"
    )


@main.command()
# TODO: Use https://github.com/googleapis/protoc-docs-plugin to add docstrings
# to the pb2 module so that I can reference them here.
@click.option("--name", required=True)
@click.option("--version", required=True)
@click.option("--language", required=True)
@click.option("--distribution-name", default="")
@click.argument("destination", default="docs.metadata")
def create_metadata(
    name: str, version: str, language: str, distribution_name: str, destination: str
):
    metadata = metadata_pb2.Metadata()
    metadata.update_time.FromDatetime(datetime.datetime.utcnow())
    metadata.name = name
    metadata.language = language
    metadata.version = version
    metadata.distribution_name = distribution_name

    destination_path = pathlib.Path(destination)
    destination_path.write_text(text_format.MessageToString(metadata))
    docuploader.log.success(f"Wrote metadata to {destination_path}.")


if __name__ == "__main__":
    main()
{'fraction_non_alphanumeric': 0.09823570924488356, 'fraction_numerical': 0.004375441072688779, 'mean_word_length': 3.5019059720457433, 'pattern_counts': {'":': 2, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 3, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6164923', 'n_tokens_mistral': 2060, 'n_tokens_neox': 1979, 'n_words': 599}