<commit_before>from __future__ import print_function
def main(args=None):

    __author__    = "<NAME> and <NAME>"
    __copyright__ = "Copyright (C) 2017 <NAME> and <NAME>"
    __license__   = "Public Domain"
    __version__   = "0.29559774252"

    flag_debug = False

    #
    ##
    ### IMPORTS
    ##
    #

    #-- Stdio manipulations, read, write and stderr
    try:
        import sys
        import os
        import os.path
        import argparse
        from argparse import RawTextHelpFormatter

    except:
        print ("Something went wrong, cannot import package 'sys' or 'os' or 'argparse'")
        exit(1)

    if sys.version_info.major != 3:
        sys.stderr.write('You are using Python 2, consider using Python 3.\n')

    #
    ##
    ###  PARSING COMMAND LINE ARGUMENTS
    ##
    #


    class bcolors:
        HEADER = '\033[95m'
        OKBLUE = '\033[94m'
        OKGREEN = '\033[92m'
        WARNING = '\033[93m'
        FAIL = '\033[91m'
        ENDC = '\033[0m'
        BOLD = '\033[1m'
        UNDERLINE = '\033[4m'

    example_string = bcolors.BOLD + """Example """ + bcolors.ENDC + """

    Consider a system for marking abstractions in lambda-terms:

    L = z L^2 + u z L + D
    D = z + z D

    We want to have 40% of abstractions, so we encode all the variables and
    functions into a single vector [z, u, L, D] and construct input file
    2 1
    0.4
    3
    1 1 1 0
    1 0 2 0
    0 0 0 1
    2
    1 0 0 0
    1 0 0 1
    """

    parser = argparse.ArgumentParser(
            description= bcolors.BOLD + """Welcome to paganini.py! """ +
            bcolors.ENDC,
            epilog = example_string,
            formatter_class=RawTextHelpFormatter)

    parser.add_argument('-i', '--input', dest='input', nargs=1,
            required=False, help="Name of the input file")
    parser.add_argument('--from-stdin', dest='from_stdin', action='store_true',
            required=False, help="Take the input from stdin")
    parser.add_argument('-s', '--solver', dest='solver', nargs=1,
            required=False, help="Solver: [CVXOPT, SCS, ECOS]. Default is ECOS.")
    parser.add_argument('-p', '--precision', dest='precision', nargs=1,
            type=float,
            required=False, help="Precision. Defaults to 1e-20.")
    parser.add_argument('-m', '--max-iters', dest='maxiters', nargs=1, type=int,
            required=False, help="Maximum number of iterations.")
    parser.add_argument('-t', '--type', dest='type',
            required=False, help="Type of the grammar: [rational, algebraic]")

    arguments = parser.parse_args()

    if arguments.from_stdin==False and arguments.input==None:
        parser.print_help(sys.stderr)
        exit(1)

    if arguments.from_stdin!=False and arguments.input!=None:
        sys.stderr.write('Choose either stdin or filename for input\n')
        parser.print_help(sys.stderr)
        exit(1)

    sys.stderr.write("Importing packages...\n")

    #
    ##
    ### IMPORT HEAVY PACKAGES
    ##
    #

    #-- Better hints at non-installed packages
    list_of_noninstalled_packages = []

    # --- cvxopt is truly necessary
    try:
        import cvxpy
    except:
        list_of_noninstalled_packages += ['cvxpy']

    # --- sympy is an insurance from exponent overflow
    # -- It will be removed for practical purposes and replaced by bounded region
    # feasible subset.
    try:
        import sympy
    except:
        list_of_noninstalled_packages += ['sympy']

    try:
        import numpy as np
    except:
        list_of_noninstalled_packages += ['numpy']

    try:
        from six.moves import range
    except:
        list_of_noninstalled_packages += ['six']

    if len(list_of_noninstalled_packages) > 0:
        sys.stderr.write("""It seems that you need to install some packages.
    Please be patient and type into your command line
        pip2 install """ + ' '.join(list_of_noninstalled_packages) + """
    If you have only Python2 installed, you can also try
        pip install """ + ' '.join(list_of_noninstalled_packages) + """
    Good luck!
    """)
        sys.exit(1)

    ### PRECISION

    np.set_printoptions(precision=14)


    sys.stderr.write("Started concerto...\n")

    filename = arguments.input[0] if arguments.input else None
    precision = 1e-20
    if arguments.precision != None:
        try:
            precision = float(arguments.precision[0])
        except:
            raise Exception("Precision should be a float!")
    is_rational = False
    if arguments.type == 'rational':
        sys.stderr.write("System is of rational type\n")
        is_rational = True
    elif arguments.type == 'algebraic':
        is_rational = False
    elif arguments.type != None:
        sys.stderr.write("Type of the grammar not recognized, using algebraic.\n")

    solver = cvxpy.ECOS
    maxiters = 20

    if arguments.solver != None:
        if (arguments.solver[0] == 'CVXOPT'):
            solver = cvxpy.CVXOPT
            maxiters = 20
        elif (arguments.solver[0] == 'SCS'):
            solver = cvxpy.SCS
            maxiters = 2500
        elif (arguments.solver[0] == 'ECOS'):
            solver = cvxpy.ECOS
            maxiters = 20
        else:
            sys.stderr.write("Solver not recognized. Using ECOS by default.\n")

    if arguments.maxiters != None:
        maxiters = int(arguments.maxiters[0])

    if filename and not os.path.isfile(filename):
        raise Exception("File doesn't exist!", filename)

    #
    ##
    ### MAIN CODE
    ##
    #

    # set default error file name
    err_filename = filename if filename else "stdin"

    input_error_string = """
    Input format error in '""" + err_filename + """'!
    Expected:
    <n_fun> <n_var>
    <freq1> <freq2> ... <freq n_var>
    <number of monomials in first definition>
    <exponent1> <exponent2> ... <exponent n_var> - for the first monomial
    <exponent1> <exponent2> ... <exponent n_var> - for the second
    ...
    <number of monomials in second definition>
    ...

    Type python2 paganini.py for help and examples.
    """

    FILE = open(filename,'r') if filename else sys.stdin

    # Read the number of variables and functions

    vec = FILE.readline().split()
    assert np.size(vec) >= 2, input_error_string
    try:
        number_of_functions = int(vec[0])
        number_of_variables = int(vec[1])
        total_number_of_variables = number_of_functions + number_of_variables + 1
    except:
        raise Exception(input_error_string)
    assert number_of_functions > 0, '\nThe number of functions should be a positive integer.\n'
    assert number_of_variables >= 0, '\nThe number of functions should be nonnegative integer.\n'

    # Read the frequences
    vec = FILE.readline().split()
    assert np.size(vec) >= number_of_variables,\
                '\nThe number of frequences should be equal to number of variables.\n'
    if number_of_variables > 0:
        try:
            freq = [float(elem) for elem in vec]
        except:
            raise Exception(input_error_string)
    else:
        freq = []

    sys.stderr.write("Reading the coefficients... ")
    # Read the coefficients of equations
    coeff_array = []
    for n_equation in range(number_of_functions):
        coeff_array += [[]]
        vec = FILE.readline().split()
        assert np.size(vec) >= 1,\
                'What is the number of monomials in equation '+ n_equation + '?\n'
        n_monomials = int(vec[0])
        for monomial in range(n_monomials):
            vec = FILE.readline().split()
            coeff_array[-1] += [[int(elem) for elem in vec]]

    sys.stderr.write("done!\n")

    sys.stderr.write("Composing optimization problem... ")

    z = cvxpy.Variable((number_of_variables + number_of_functions + 1))

    obj = np.array( [[1.0] + freq + [0.0] *  number_of_functions])

    constraints = [
                z[idx + number_of_variables + 1] >=
                cvxpy.log_sum_exp(np.array(coeff_array[idx]) * z)
                for idx in range(number_of_functions)
            ]
    if is_rational:
        constraints += [
            cvxpy.norm(z, 2) <= 40
        ]<commit_msg>IMP: Sparse input format for paganini<commit_after>from __future__ import print_function
def main(args=None):

    __author__    = "<NAME> and <NAME>"
    __copyright__ = "Copyright (C) 2017-2019 <NAME> and <NAME>"
    __license__   = "Public Domain"
    __version__   = "0.29559774252"

    flag_debug = False

    #
    ##
    ### IMPORTS
    ##
    #

    #-- Stdio manipulations, read, write and stderr
    try:
        import sys
        import os
        import os.path
        import argparse
        from argparse import RawTextHelpFormatter

    except:
        print ("Something went wrong, cannot import packages 'sys', 'os' or 'argparse'.")
        exit(1)

    if sys.version_info.major != 3:
        sys.stderr.write('You are using Python 2, consider using Python 3.\n')

    #
    ##
    ###  PARSING COMMAND LINE ARGUMENTS
    ##
    #

    class bcolors:
        HEADER = '\033[95m'
        OKBLUE = '\033[94m'
        OKGREEN = '\033[92m'
        WARNING = '\033[93m'
        FAIL = '\033[91m'
        ENDC = '\033[0m'
        BOLD = '\033[1m'
        UNDERLINE = '\033[4m'

    example_string = bcolors.BOLD + """Example """ + bcolors.ENDC + """

    Consider a system for marking abstractions in lambda-terms:

    L = z L^2 + u z L + D
    D = z + z D

    We want to have 40% of abstractions, so we encode all the variables and
    functions into a single vector [z, u, L, D] and construct input file
    2 1
    0.4
    3
    1 1 1 0
    1 0 2 0
    0 0 0 1
    2
    1 0 0 0
    1 0 0 1
    """

    parser = argparse.ArgumentParser(
            description= bcolors.BOLD + """Welcome to paganini.py! """ +
            bcolors.ENDC,
            epilog = example_string,
            formatter_class=RawTextHelpFormatter)

    parser.add_argument('-i', '--input', dest='input', nargs=1,
            required=False, help="Name of the input file")
    parser.add_argument('--from-stdin', dest='from_stdin', action='store_true',
            required=False, help="Take the input from stdin")
    parser.add_argument('-s', '--solver', dest='solver', nargs=1,
            required=False, help="Solver: [CVXOPT, SCS, ECOS]. Default is ECOS.")
    parser.add_argument('-p', '--precision', dest='precision', nargs=1,
            type=float,
            required=False, help="Precision. Defaults to 1e-20.")
    parser.add_argument('-m', '--max-iters', dest='maxiters', nargs=1, type=int,
            required=False, help="Maximum number of iterations.")
    parser.add_argument('-t', '--type', dest='type',
            required=False, help="Type of the grammar: [rational, algebraic]")

    arguments = parser.parse_args()

    if arguments.from_stdin==False and arguments.input==None:
        parser.print_help(sys.stderr)
        exit(1)

    if arguments.from_stdin!=False and arguments.input!=None:
        sys.stderr.write('Choose either stdin or filename for input\n')
        parser.print_help(sys.stderr)
        exit(1)

    sys.stderr.write("Importing packages...\n")

    #
    ##
    ### IMPORT HEAVY PACKAGES
    ##
    #

    #-- Better hints at non-installed packages
    list_of_noninstalled_packages = []

    # --- cvxopt is truly necessary
    try:
        import cvxpy
    except:
        list_of_noninstalled_packages += ['cvxpy']

    # --- sympy is an insurance from exponent overflow
    # -- It will be removed for practical purposes and replaced by bounded region
    # feasible subset.
    try:
        import sympy
    except:
        list_of_noninstalled_packages += ['sympy']

    try:
        import numpy as np
    except:
        list_of_noninstalled_packages += ['numpy']

    try:
        from six.moves import range
    except:
        list_of_noninstalled_packages += ['six']

    if len(list_of_noninstalled_packages) > 0:
        sys.stderr.write("""It seems that you need to install some packages.
    Please be patient and type into your command line
        pip2 install """ + ' '.join(list_of_noninstalled_packages) + """
    If you have only Python2 installed, you can also try
        pip install """ + ' '.join(list_of_noninstalled_packages) + """
    Good luck!
    """)
        sys.exit(1)

    ### PRECISION

    np.set_printoptions(precision=14)


    sys.stderr.write("Started concerto...\n")

    filename = arguments.input[0] if arguments.input else None
    precision = 1e-20
    if arguments.precision != None:
        try:
            precision = float(arguments.precision[0])
        except:
            raise Exception("Precision should be a float!")
    is_rational = False
    if arguments.type == 'rational':
        sys.stderr.write("System is of rational type\n")
        is_rational = True
    elif arguments.type == 'algebraic':
        is_rational = False
    elif arguments.type != None:
        sys.stderr.write("Type of the grammar not recognized, using algebraic.\n")

    solver = cvxpy.ECOS
    maxiters = 20

    if arguments.solver != None:
        if (arguments.solver[0] == 'CVXOPT'):
            solver = cvxpy.CVXOPT
            maxiters = 20
        elif (arguments.solver[0] == 'SCS'):
            solver = cvxpy.SCS
            maxiters = 2500
        elif (arguments.solver[0] == 'ECOS'):
            solver = cvxpy.ECOS
            maxiters = 20
        else:
            sys.stderr.write("Solver not recognized. Using ECOS by default.\n")

    if arguments.maxiters != None:
        maxiters = int(arguments.maxiters[0])

    if filename and not os.path.isfile(filename):
        raise Exception("File doesn't exist!", filename)

    #
    ##
    ### MAIN CODE
    ##
    #

    # set default error file name
    err_filename = filename if filename else "stdin"

    input_error_string = """
    Input format error in '""" + err_filename + """'!
    Expected:
    <n_fun> <n_var>
    <freq1> <freq2> ... <freq n_var>
    <number of monomials in first definition>
    <exponent1> <exponent2> ... <exponent n_var> - for the first monomial
    <exponent1> <exponent2> ... <exponent n_var> - for the second
    ...
    <number of monomials in second definition>
    ...

    Type python2 paganini.py for help and examples.
    """

    FILE = open(filename,'r') if filename else sys.stdin

    # Read the number of variables and functions

    vec = FILE.readline().split()
    assert np.size(vec) >= 2, input_error_string
    try:
        number_of_functions = int(vec[0])
        number_of_variables = int(vec[1])
        total_number_of_variables = number_of_functions + number_of_variables + 1
    except:
        raise Exception(input_error_string)
    assert number_of_functions > 0, '\nThe number of functions should be a positive integer.\n'
    assert number_of_variables >= 0, '\nThe number of functions should be nonnegative integer.\n'

    # Read the frequences
    vec = FILE.readline().split()
    assert np.size(vec) >= number_of_variables,\
                '\nThe number of frequences should be equal to number of variables.\n'
    if number_of_variables > 0:
        try:
            freq = [float(elem) for elem in vec]
        except:
            raise Exception(input_error_string)
    else:
        freq = []

    def unpack(vec):
        """ Given a splitted text line, unpacks the sparse vector representation. """
        res = [0] * total_number_of_variables
        for elem in vec:
            arr = elem[1:-1].split(',')
            res[int(arr[1])] = int(arr[0])
        return res

    sys.stderr.write("Reading the coefficients... ")
    # Read the coefficients of equations
    coeff_array = []
    for n_equation in range(number_of_functions):
        coeff_array += [[]]
        vec = FILE.readline().split()
        assert np.size(vec) >= 1,\
                'What is the number of monomials in equation '+ n_equation + '?\n'
        n_monomials = int(vec[0])
        for monomial in range(n_monomials):
            vec = FILE.readline().split()
            coeff_array[-1] += [[int(elem) for elem in unpack(vec)]]

    sys.stderr.write("done!\n")

    sys.stderr.write("Composing optimization problem... ")

    z = cvxpy.Variable((number_of_variables + number_of_functions + 1))

    obj = np.array( [[1.0] + freq + [0.0] *  number_of_functions])

    constraints = [
                z[idx + number_of_variables + 1] >=
                cvxpy.log_sum_exp(np.array(coeff_array[idx]) * z)
                for idx in range(number_of_functions)
            ]
    if is_rational:
        constraints += [
            cvxpy.norm(z, 2) <= 40
        ]
{'fraction_non_alphanumeric': 0.11689014491010763, 'fraction_numerical': 0.018098731284949793, 'mean_word_length': 2.2695105170041283, 'pattern_counts': {'":': 0, '<': 39, '<?xml version=': 0, '>': 53, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '335608', 'n_tokens_mistral': 5426, 'n_tokens_neox': 5024, 'n_words': 1409}