<commit_before>#!/usr/bin/env python3

import click
import json
import os
import yaml
import requests
import gzip
import urllib
import shutil
import re
import glob
import logging
import sys
import traceback

import yamldown

from functools import wraps

# from ontobio.util.user_agent import get_user_agent
from ontobio.ontol_factory import OntologyFactory
from ontobio.io.gafparser import GafParser
from ontobio.io.gpadparser import GpadParser
from ontobio.io.assocwriter import GafWriter
from ontobio.io.assocwriter import GpadWriter
from ontobio.io import assocparser
from ontobio.io import gafgpibridge
from ontobio.io import entitywriter
from ontobio.io import gaference
from ontobio.rdfgen import assoc_rdfgen
from ontobio.validation import metadata
from ontobio.validation import tools
from ontobio.validation import rules

from typing import Dict, Set

# logging.basicConfig(format="%(asctime)s - %(name)s - %(levelname)s: %(message)s", level=logging.WARNING)

logger = logging.getLogger("ontobio")

def thispath():
    os.path.normpath(os.path.abspath(__file__))


def download_a_dataset_source(group, dataset_metadata, target_dir, source_url, base_download_url=None, replace_existing_files=True):
    """
    This will download a dataset source given the group name,
    the metadata stanza for the dataset, and the target directory that all downloads
    occur in.

    The path will be built from these elements and then the URL will be found
    from which to download the file.

    `base_download_url` if set will change the URL to a local relative path, rather
    than a true download. This means the dataset_metadata could have "relative/path/to/gaf"
    and the absolute on disk path will be constructed by appending the found metadata
    path to `base_download_url`.

    `replace_existing_files` by default is True. This will overwrite any existing file, always
    updating. With `replace_existing_files` False the path will be checked if a file already
    exists there and if so the actual download will not proceed. The found file
    will be assumed to be the correct file.
    """
    # Local target download path setup - path and then directories
    file_name = source_url.split("/")[-1]
    path = metadata.source_path(dataset_metadata, target_dir, group)

    # Just return the path if we find that it exists already
    if os.path.exists(path) and not replace_existing_files:
        click.echo("{} already exists, no need to download - skipping".format(path))
        return path

    os.makedirs(os.path.split(path)[0], exist_ok=True)

    click.echo("Downloading source to {}".format(path))

    # Parse urls
    source_url_parsed = urllib.parse.urlparse(source_url)
    base_download_url_parsed = urllib.parse.urlparse(base_download_url) if base_download_url != None else None

    # Compute scheme:
    scheme = "file"
    if source_url_parsed.scheme != "":
        scheme = source_url_parsed.scheme
    elif base_download_url_parsed.scheme != "":
        scheme = base_download_url_parsed.scheme

    # Compute joined url, including dealing with relative paths
    joined_url = source_url
    if not source_url_parsed.path.startswith("/"):
        # We're relative
        if base_download_url is not None:
            joined_url = urllib.parse.urljoin(base_download_url, source_url)
        else:
            # This is bad and we have to jump out`since we have no way of constructing a real gaf url
            raise click.ClickException("Option `--base-download-url` was not specified and the config url {} is a relative path.".format(source_url))

    # including scheme and such
    reconstructed_url = urllib.parse.urlunsplit((scheme, urllib.parse.urlparse(joined_url).netloc, urllib.parse.urlparse(joined_url).path, source_url_parsed.query, ""))

    # Using urllib to download if scheme is ftp or file. Otherwise we can use requests and use a progressbar
    if scheme in ["ftp", "file"]:
        urllib.request.urlretrieve(reconstructed_url, path)
    else:
        response = requests.get(reconstructed_url, stream=True)
        content_length = int(response.headers.get("Content-Length", 0))

        with open(path, "wb") as downloaded:
            with click.progressbar(iterable=response.iter_content(chunk_size=512 * 1024), length=content_length, show_percent=True) as chunks:
                for chunk in chunks:
                    if chunk:
                        downloaded.write(chunk)

    return path

def download_source_gafs(group_metadata, target_dir, exclusions=[], base_download_url=None, replace_existing_files=True, only_dataset=None):
    """
    This looks at a group metadata dictionary and downloads each GAF source that is not in the exclusions list.
    For each downloaded file, keep track of the path of the file. If the file is zipped, it will unzip it here.
    This function returns a list of tuples of the dataset dictionary mapped to the downloaded source path.
    """
    # Grab all datasets in a group, excluding non-gaf, datasets that are explicitely excluded from an option, and excluding datasets with the `exclude key` set to true
    gaf_urls = []
    if only_dataset is None:
        gaf_urls = [ (data, data["source"]) for data in group_metadata["datasets"] if data["type"] == "gaf" and data["dataset"] not in exclusions and not data.get("exclude", False)]
    else:
        gaf_urls = [ (data, data["source"]) for data in group_metadata["datasets"] if data["dataset"] == only_dataset ]
    # List of dataset metadata to gaf download url

    click.echo("Found {}".format(", ".join( [ kv[0]["dataset"] for kv in gaf_urls ] )))
    downloaded_paths = []
    for dataset_metadata, gaf_url in gaf_urls:
        dataset = dataset_metadata["dataset"]
        # Local target download path setup - path and then directories
        path = download_a_dataset_source(group_metadata["id"], dataset_metadata, target_dir, gaf_url, base_download_url=base_download_url, replace_existing_files=replace_existing_files)

        if dataset_metadata["compression"] == "gzip":
            # Unzip any downloaded file that has gzip, strip of the gzip extension
            unzipped = os.path.splitext(path)[0]
            unzip(path, unzipped)
            path = unzipped
        else:
            # otherwise file is coming in uncompressed. But we want to make sure
            # to zip up the original source also
            tools.zipup(path)

        downloaded_paths.append((dataset_metadata, path))

    return downloaded_paths

def check_and_download_mixin_source(mixin_metadata, group_id, dataset, target_dir, base_download_url=None, replace_existing_files=True):
    mixin_dataset = tools.find(mixin_metadata["datasets"], lambda d: d.get("merges_into", "") == dataset)
    if mixin_dataset is None:
        return None

    click.echo("Merging mixin dataset {}".format(mixin_dataset["source"]))
    path = download_a_dataset_source(group_id, mixin_dataset, target_dir, mixin_dataset["source"], base_download_url=base_download_url, replace_existing_files=replace_existing_files)

    unzipped = os.path.splitext(path)[0] # Strip off the .gz extension, leaving just the unzipped filename
    unzip(path, unzipped)
    return unzipped

def mixin_dataset(mixin_metadata, dataset):
    mixin_dataset_version = tools.find(mixin_metadata["datasets"], lambda d: d.get("merges_into", "") == dataset)
    if mixin_dataset_version is None:
        return None # Blah
<commit_msg>protect against a missing key in dataset yaml<commit_after>#!/usr/bin/env python3

import click
import json
import os
import yaml
import requests
import gzip
import urllib
import shutil
import re
import glob
import logging
import sys
import traceback

import yamldown

from functools import wraps

# from ontobio.util.user_agent import get_user_agent
from ontobio.ontol_factory import OntologyFactory
from ontobio.io.gafparser import GafParser
from ontobio.io.gpadparser import GpadParser
from ontobio.io.assocwriter import GafWriter
from ontobio.io.assocwriter import GpadWriter
from ontobio.io import assocparser
from ontobio.io import gafgpibridge
from ontobio.io import entitywriter
from ontobio.io import gaference
from ontobio.rdfgen import assoc_rdfgen
from ontobio.validation import metadata
from ontobio.validation import tools
from ontobio.validation import rules

from typing import Dict, Set

# logging.basicConfig(format="%(asctime)s - %(name)s - %(levelname)s: %(message)s", level=logging.WARNING)

logger = logging.getLogger("ontobio")

def thispath():
    os.path.normpath(os.path.abspath(__file__))


def download_a_dataset_source(group, dataset_metadata, target_dir, source_url, base_download_url=None, replace_existing_files=True):
    """
    This will download a dataset source given the group name,
    the metadata stanza for the dataset, and the target directory that all downloads
    occur in.

    The path will be built from these elements and then the URL will be found
    from which to download the file.

    `base_download_url` if set will change the URL to a local relative path, rather
    than a true download. This means the dataset_metadata could have "relative/path/to/gaf"
    and the absolute on disk path will be constructed by appending the found metadata
    path to `base_download_url`.

    `replace_existing_files` by default is True. This will overwrite any existing file, always
    updating. With `replace_existing_files` False the path will be checked if a file already
    exists there and if so the actual download will not proceed. The found file
    will be assumed to be the correct file.
    """
    # Local target download path setup - path and then directories
    file_name = source_url.split("/")[-1]
    path = metadata.source_path(dataset_metadata, target_dir, group)

    # Just return the path if we find that it exists already
    if os.path.exists(path) and not replace_existing_files:
        click.echo("{} already exists, no need to download - skipping".format(path))
        return path

    os.makedirs(os.path.split(path)[0], exist_ok=True)

    click.echo("Downloading source to {}".format(path))

    # Parse urls
    source_url_parsed = urllib.parse.urlparse(source_url)
    base_download_url_parsed = urllib.parse.urlparse(base_download_url) if base_download_url != None else None

    # Compute scheme:
    scheme = "file"
    if source_url_parsed.scheme != "":
        scheme = source_url_parsed.scheme
    elif base_download_url_parsed.scheme != "":
        scheme = base_download_url_parsed.scheme

    # Compute joined url, including dealing with relative paths
    joined_url = source_url
    if not source_url_parsed.path.startswith("/"):
        # We're relative
        if base_download_url is not None:
            joined_url = urllib.parse.urljoin(base_download_url, source_url)
        else:
            # This is bad and we have to jump out`since we have no way of constructing a real gaf url
            raise click.ClickException("Option `--base-download-url` was not specified and the config url {} is a relative path.".format(source_url))

    # including scheme and such
    reconstructed_url = urllib.parse.urlunsplit((scheme, urllib.parse.urlparse(joined_url).netloc, urllib.parse.urlparse(joined_url).path, source_url_parsed.query, ""))

    # Using urllib to download if scheme is ftp or file. Otherwise we can use requests and use a progressbar
    if scheme in ["ftp", "file"]:
        urllib.request.urlretrieve(reconstructed_url, path)
    else:
        response = requests.get(reconstructed_url, stream=True)
        content_length = int(response.headers.get("Content-Length", 0))

        with open(path, "wb") as downloaded:
            with click.progressbar(iterable=response.iter_content(chunk_size=512 * 1024), length=content_length, show_percent=True) as chunks:
                for chunk in chunks:
                    if chunk:
                        downloaded.write(chunk)

    return path

def download_source_gafs(group_metadata, target_dir, exclusions=[], base_download_url=None, replace_existing_files=True, only_dataset=None):
    """
    This looks at a group metadata dictionary and downloads each GAF source that is not in the exclusions list.
    For each downloaded file, keep track of the path of the file. If the file is zipped, it will unzip it here.
    This function returns a list of tuples of the dataset dictionary mapped to the downloaded source path.
    """
    # Grab all datasets in a group, excluding non-gaf, datasets that are explicitely excluded from an option, and excluding datasets with the `exclude key` set to true
    gaf_urls = []
    if only_dataset is None:
        gaf_urls = [ (data, data["source"]) for data in group_metadata["datasets"] if data["type"] == "gaf" and data["dataset"] not in exclusions and not data.get("exclude", False)]
    else:
        gaf_urls = [ (data, data["source"]) for data in group_metadata["datasets"] if data["dataset"] == only_dataset ]
    # List of dataset metadata to gaf download url

    click.echo("Found {}".format(", ".join( [ kv[0]["dataset"] for kv in gaf_urls ] )))
    downloaded_paths = []
    for dataset_metadata, gaf_url in gaf_urls:
        dataset = dataset_metadata["dataset"]
        # Local target download path setup - path and then directories
        path = download_a_dataset_source(group_metadata["id"], dataset_metadata, target_dir, gaf_url, base_download_url=base_download_url, replace_existing_files=replace_existing_files)

        if dataset_metadata.get("compression", None) == "gzip":
            # Unzip any downloaded file that has gzip, strip of the gzip extension
            unzipped = os.path.splitext(path)[0]
            unzip(path, unzipped)
            path = unzipped
        else:
            # otherwise file is coming in uncompressed. But we want to make sure
            # to zip up the original source also
            tools.zipup(path)

        downloaded_paths.append((dataset_metadata, path))

    return downloaded_paths

def check_and_download_mixin_source(mixin_metadata, group_id, dataset, target_dir, base_download_url=None, replace_existing_files=True):
    mixin_dataset = tools.find(mixin_metadata["datasets"], lambda d: d.get("merges_into", "") == dataset)
    if mixin_dataset is None:
        return None

    click.echo("Merging mixin dataset {}".format(mixin_dataset["source"]))
    path = download_a_dataset_source(group_id, mixin_dataset, target_dir, mixin_dataset["source"], base_download_url=base_download_url, replace_existing_files=replace_existing_files)

    unzipped = os.path.splitext(path)[0] # Strip off the .gz extension, leaving just the unzipped filename
    unzip(path, unzipped)
    return unzipped

def mixin_dataset(mixin_metadata, dataset):
    mixin_dataset_version = tools.find(mixin_metadata["datasets"], lambda d: d.get("merges_into", "") == dataset)
    if mixin_dataset_version is None:
        return None # Blah

{'fraction_non_alphanumeric': 0.08046592582675056, 'fraction_numerical': 0.0018744142455482662, 'mean_word_length': 3.8159252095422307, 'pattern_counts': {'":': 6, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 84, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1625189', 'n_tokens_mistral': 4334, 'n_tokens_neox': 4287, 'n_words': 1428}