<commit_before>#This is an example of downloading a repo zip file, 
#unzipping it and running a scan on it.
#The example uses scancode, but dosocsv2 could also be used.

import subprocess
from subprocess import call
import requests
import os
from os import path, remove
import shutil
import zipfile
import sys
import click
from yaml import safe_load, dump

@click.command()
@click.option('--url', prompt='The url of the GitHub repo zip file to scan', 
              help='The url of the GitHub repo zip file to scan.')

def main(url):
    #Argument is the url of the GitHub zip file
    repo_zip_url = url
    #If url gives error, exit
    if(check_valid_url(repo_zip_url) == False):
        sys.exit()
    repo_scan(repo_zip_url)

#The overall process of downloading and scanning a repo
def repo_scan(repo_zip_url):
    #Download the zip and get its path
    file_location = download_github_zip(repo_zip_url)
    #extract the zip and get path of extracted directory
    extracted_directory = unzip_file(file_location)

    config_file = extracted_directory + 'configuration.YAML'

    configuration = get_config(config_file)
    suffix = configuration['output_file_name'][-5:]
    if(suffix != '.SPDX' and suffix != '.spdx'):
        spdx_file_name = configuration['output_file_name'] + '.SPDX'
    else:
        spdx_file_name = configuration['output_file_name']

    #scan the extracted directory and put results in a named file
    scan(extracted_directory, spdx_file_name, 'scancode', 
         configuration['output_type'])

    #Remove the zip file
    remove(file_location)
    #Remove the unzipped directory
    shutil.rmtree(extracted_directory)
    
    return spdx_file_name

def scan(directory_to_scan, output_file_name, scanner, output_type):
    if(output_type == 'tag-value'):
        output_format = 'spdx-tv'
    elif(output_type == 'rdf'):
        output_format = 'spdx-rdf'
    #Scan the unzipped directory
    print subprocess.check_output(['scancode','--format', 
                                  output_format,directory_to_scan,
                                  output_file_name])

#Unzip a zip file and return the path to the extracted directory
def unzip_file(file_location):
    #Unzip the zip file
    zip_file = zipfile.ZipFile(file_location, 'r')
    zip_file.extractall()
    #Get the directory that results from unzipping
    #(Assume there is only one)
    extracted_directory = (zip_file.namelist())[0]
    zip_file.close()
    #return the path to the extracted directory
    return extracted_directory


#Download a zip file and return the path of the downloaded file
def download_github_zip(repo_zip_url):
    #Getting a zip file to download
    to_scan = requests.get(repo_zip_url, stream=True)

    #Get path to where zip file will be stored
    #Name of the zip file
    zip_file_name = repo_zip_url[repo_zip_url.rfind('/')+1:]
    #Path to the directory containing the zip file
    directory_path = os.path.dirname(os.path.realpath(__file__))
    #Concatenate file name with path
    file_location = directory_path + zip_file_name

    #write zip file in chunks
    with open(file_location, 'wb') as fd:
        for chunk in to_scan.iter_content(chunk_size=1024):
            fd.write(chunk)
    #return path to the downloaded file
    return file_location    

def get_config(config_file):
    stream = file(config_file, 'r')
    configuration = safe_load(stream)
    return configuration

#Check that the url for the zip file can be reached
def check_valid_url(repo_zip_url):
    #get just the head
    request = requests.head(repo_zip_url)
    #It should be success (200's) or redirect(300's)
    #Otherwise, inform user of failure
    if (request.status_code >= 400 or request.status_code < 200):
        print('Could not reach URL provided.\n')
        print('Provided url was ' + repo_zip_url + 
              ' and resulted in status code ' + str(request.status_code))
        return False
    else:
        return True

if __name__ == "__main__": main()
<commit_msg>line break before operator<commit_after>#This is an example of downloading a repo zip file, 
#unzipping it and running a scan on it.
#The example uses scancode, but dosocsv2 could also be used.

import subprocess
from subprocess import call
import requests
import os
from os import path, remove
import shutil
import zipfile
import sys
import click
from yaml import safe_load, dump

@click.command()
@click.option('--url', prompt='The url of the GitHub repo zip file to scan', 
              help='The url of the GitHub repo zip file to scan.')

def main(url):
    #Argument is the url of the GitHub zip file
    repo_zip_url = url
    #If url gives error, exit
    if(check_valid_url(repo_zip_url) == False):
        sys.exit()
    repo_scan(repo_zip_url)

#The overall process of downloading and scanning a repo
def repo_scan(repo_zip_url):
    #Download the zip and get its path
    file_location = download_github_zip(repo_zip_url)
    #extract the zip and get path of extracted directory
    extracted_directory = unzip_file(file_location)

    config_file = extracted_directory + 'configuration.YAML'

    configuration = get_config(config_file)
    suffix = configuration['output_file_name'][-5:]
    if(suffix != '.SPDX' and suffix != '.spdx'):
        spdx_file_name = configuration['output_file_name'] + '.SPDX'
    else:
        spdx_file_name = configuration['output_file_name']

    #scan the extracted directory and put results in a named file
    scan(extracted_directory, spdx_file_name, 'scancode', 
         configuration['output_type'])

    #Remove the zip file
    remove(file_location)
    #Remove the unzipped directory
    shutil.rmtree(extracted_directory)
    
    return spdx_file_name

def scan(directory_to_scan, output_file_name, scanner, output_type):
    if(output_type == 'tag-value'):
        output_format = 'spdx-tv'
    elif(output_type == 'rdf'):
        output_format = 'spdx-rdf'
    #Scan the unzipped directory
    print subprocess.check_output(['scancode','--format', 
                                  output_format,directory_to_scan,
                                  output_file_name])

#Unzip a zip file and return the path to the extracted directory
def unzip_file(file_location):
    #Unzip the zip file
    zip_file = zipfile.ZipFile(file_location, 'r')
    zip_file.extractall()
    #Get the directory that results from unzipping
    #(Assume there is only one)
    extracted_directory = (zip_file.namelist())[0]
    zip_file.close()
    #return the path to the extracted directory
    return extracted_directory


#Download a zip file and return the path of the downloaded file
def download_github_zip(repo_zip_url):
    #Getting a zip file to download
    to_scan = requests.get(repo_zip_url, stream=True)

    #Get path to where zip file will be stored
    #Name of the zip file
    zip_file_name = repo_zip_url[repo_zip_url.rfind('/')+1:]
    #Path to the directory containing the zip file
    directory_path = os.path.dirname(os.path.realpath(__file__))
    #Concatenate file name with path
    file_location = directory_path + zip_file_name

    #write zip file in chunks
    with open(file_location, 'wb') as fd:
        for chunk in to_scan.iter_content(chunk_size=1024):
            fd.write(chunk)
    #return path to the downloaded file
    return file_location    

def get_config(config_file):
    stream = file(config_file, 'r')
    configuration = safe_load(stream)
    return configuration

#Check that the url for the zip file can be reached
def check_valid_url(repo_zip_url):
    #get just the head
    request = requests.head(repo_zip_url)
    #It should be success (200's) or redirect(300's)
    #Otherwise, inform user of failure
    if (request.status_code >= 400 or request.status_code < 200):
        print('Could not reach URL provided.\n')
        print('Provided url was ' + repo_zip_url 
              + ' and resulted in status code ' + str(request.status_code))
        return False
    else:
        return True

if __name__ == "__main__": main()

{'fraction_non_alphanumeric': 0.07982096232748974, 'fraction_numerical': 0.004973268680840482, 'mean_word_length': 3.3622559652928414, 'pattern_counts': {'":': 2, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 22, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3900212', 'n_tokens_mistral': 2450, 'n_tokens_neox': 2357, 'n_words': 789}