<commit_before>import json
import os
import re
import urlparse

from impactlib.github import GitHub
from impactlib.semver import SemanticVersion
from impactlib import config

def extract_dependencies(fp):
    deps = {}
    contents = fp.read()
    contents = contents.replace("\n","")
    contents = contents.replace("\r","")
    pat = """([A-Za-z]\w*)\s*\(\s*version\s*=\s*"(\d+\.\d+)(\.\d+)?"\s*\)"""
    c = re.compile(pat)
    matches = re.findall(c, contents)
    for m in matches:
        if m[0]=="uses":
            continue
        deps[m[0]] = m[1]+m[2]
    ret = []
    for dep in deps:
        ret.append((dep, deps[dep]))
    return ret

def get_package_details(user, repo, tag, github, ver, verbose):
    """
    This function returns a tuple.

    The first element in the tuple is that path of the actual Modelica
    package.  This tells us what, after extracting the zipball, needs
    to be moved to the install path.

    The second element in the tuple is the list of dependencies.
    """
    root = github.getRawFile(user, repo, tag, "package.mo")
    if root!=None:
        deps = extract_dependencies(root)
        root.close()
        return (".", deps)
    elif verbose:
        print "Not in root directory"
    unver_dir = github.getRawFile(user, repo, tag, repo+"/package.mo")
    if unver_dir!=None:
        deps = extract_dependencies(unver_dir)
        unver_dir.close()
        return (repo, deps)
    elif verbose:
        print "Not in unversioned directory of the same name"
    ver_name = repo+" "+str(tag.replace("v",""))
    path = repo+"%20"+str(tag.replace("v", ""))+"/package.mo"
    ver_dir = github.getRawFile(user, repo, tag, path)
    if ver_dir!=None:
        deps = extract_dependencies(ver_dir)
        ver_dir.close()
        return (ver_name, deps)
    elif verbose:
        print "Not in versioned directory ("+ver_name+")"
    return (None, [])

def process_github_user(repo_data, user, pat, github, verbose,
                        tolerant, ignore_empty):
    # Get a list of repositories
    repos = github.getRepos(user)

    c = re.compile(pat)

    # Iterate over each repository
    for repo in repos:
        # Extract the repository name
        name = repo["name"]
        if c.match(name)==None:
            continue
        print "Repository: "+name

        # Initialize data for current repository
        data = {}

        # Pull out various pieces of information about the repository
        # and store it.
        data["description"] = repo["description"]

        # If homepage field exist store this otherwise use repo home
        data["homepage"] = repo["homepage"] or repo["html_url"]

        # Prepare to extract all versions of this library
        data["versions"] = {}

        # Get the list of tags from GitHub
        tags = github.getTags(user, name)

        # Iterate over each tag
        for tag in tags:
            # Get the name of the tag
            tagname = tag["name"]
            if verbose:
                print "  Tag: "+tagname

            # Parse the tag to see if it is a semantic version number
            try:
                ver = SemanticVersion(tagname, tolerant=tolerant)
            except ValueError as e:
                if verbose:
                    print "Exception: "+str(e)
                continue

            # TODO: extract dependency information
            (path, deps) = get_package_details(user, name, tagname,
                                               github, ver, verbose)
            if path==None:
                print "Couldn't find Modelica package root"
                continue

            print "  Semantic version info: "+str(ver)
            print "    Path: "+str(path)
            print "    Dependencies: "+str(deps)

            # Create a data structure for information related to this version
            tagurlbase = ('https://github.com/%s/%s/archive/%s'
                                      % (str(user), str(name), str(tagname)))
            tagdata = ver.json()
            tagdata["zipball_url"] = tagurlbase+".zip"
            tagdata["tarball_url"] = tagurlbase+".tar.gz"
            tagdata["path"] = path
            tagdata["dependencies"] = deps

            data["versions"][str(ver)] = tagdata
            # Useful for legacy (non-semver) versions
            tver = tagname.replace("v","")
            if str(ver)!=tver:
                if verbose:
                    print "  Also storing under version: "+tver
                data["versions"][tver] = tagdata

        if len(data["versions"])==0:
            print "  No useable version tags found"
            if ignore_empty:
                continue

        # Add data for this repository to master data structure
        repo_data[name] = data<commit_msg>fixes #27 by only removing a leading "v" in a tag string<commit_after>import json
import os
import re
import urlparse

from impactlib.github import GitHub
from impactlib.semver import SemanticVersion
from impactlib import config

def extract_dependencies(fp):
    deps = {}
    contents = fp.read()
    contents = contents.replace("\n","")
    contents = contents.replace("\r","")
    pat = """([A-Za-z]\w*)\s*\(\s*version\s*=\s*"(\d+\.\d+)(\.\d+)?"\s*\)"""
    c = re.compile(pat)
    matches = re.findall(c, contents)
    for m in matches:
        if m[0]=="uses":
            continue
        deps[m[0]] = m[1]+m[2]
    ret = []
    for dep in deps:
        ret.append((dep, deps[dep]))
    return ret

def get_package_details(user, repo, tag, github, ver, verbose):
    """
    This function returns a tuple.

    The first element in the tuple is that path of the actual Modelica
    package.  This tells us what, after extracting the zipball, needs
    to be moved to the install path.

    The second element in the tuple is the list of dependencies.
    """
    root = github.getRawFile(user, repo, tag, "package.mo")
    if root!=None:
        deps = extract_dependencies(root)
        root.close()
        return (".", deps)
    elif verbose:
        print "Not in root directory"
    unver_dir = github.getRawFile(user, repo, tag, repo+"/package.mo")
    if unver_dir!=None:
        deps = extract_dependencies(unver_dir)
        unver_dir.close()
        return (repo, deps)
    elif verbose:
        print "Not in unversioned directory of the same name"
    ver_name = repo+" "+str(tag.replace("v",""))
    path = repo+"%20"+str(tag.replace("v", ""))+"/package.mo"
    ver_dir = github.getRawFile(user, repo, tag, path)
    if ver_dir!=None:
        deps = extract_dependencies(ver_dir)
        ver_dir.close()
        return (ver_name, deps)
    elif verbose:
        print "Not in versioned directory ("+ver_name+")"
    return (None, [])

def process_github_user(repo_data, user, pat, github, verbose,
                        tolerant, ignore_empty):
    # Get a list of repositories
    repos = github.getRepos(user)

    c = re.compile(pat)

    # Iterate over each repository
    for repo in repos:
        # Extract the repository name
        name = repo["name"]
        if c.match(name)==None:
            continue
        print "Repository: "+name

        # Initialize data for current repository
        data = {}

        # Pull out various pieces of information about the repository
        # and store it.
        data["description"] = repo["description"]

        # If homepage field exist store this otherwise use repo home
        data["homepage"] = repo["homepage"] or repo["html_url"]

        # Prepare to extract all versions of this library
        data["versions"] = {}

        # Get the list of tags from GitHub
        tags = github.getTags(user, name)

        # Iterate over each tag
        for tag in tags:
            # Get the name of the tag
            tagname = tag["name"]
            if verbose:
                print "  Tag: "+tagname

            # Parse the tag to see if it is a semantic version number
            try:
                ver = SemanticVersion(tagname, tolerant=tolerant)
            except ValueError as e:
                if verbose:
                    print "Exception: "+str(e)
                continue

            # TODO: extract dependency information
            (path, deps) = get_package_details(user, name, tagname,
                                               github, ver, verbose)
            if path==None:
                print "Couldn't find Modelica package root"
                continue

            print "  Semantic version info: "+str(ver)
            print "    Path: "+str(path)
            print "    Dependencies: "+str(deps)

            # Create a data structure for information related to this version
            tagurlbase = ('https://github.com/%s/%s/archive/%s'
                                      % (str(user), str(name), str(tagname)))
            tagdata = ver.json()
            tagdata["zipball_url"] = tagurlbase+".zip"
            tagdata["tarball_url"] = tagurlbase+".tar.gz"
            tagdata["path"] = path
            tagdata["dependencies"] = deps

            data["versions"][str(ver)] = tagdata
            # Useful for legacy (non-semver) versions
            tver = tagname.replace("^v","")
            if str(ver)!=tver:
                if verbose:
                    print "  Also storing under version: "+tver
                data["versions"][tver] = tagdata

        if len(data["versions"])==0:
            print "  No useable version tags found"
            if ignore_empty:
                continue

        # Add data for this repository to master data structure
        repo_data[name] = data
{'fraction_non_alphanumeric': 0.10986267166042447, 'fraction_numerical': 0.0016645859342488557, 'mean_word_length': 1.9633168927250308, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3265269', 'n_tokens_mistral': 2776, 'n_tokens_neox': 2644, 'n_words': 842}