text:
import json

import numpy as np

import benchml as bml
import benchml.transforms as btf
from benchml.hyper import GridHyper, Hyper

log = bml.log
log.setLevel("info")
bml.splits.synchronize(0)
regularization_range = np.logspace(-4, +0, 5)

def build_models():
    return [
        build_gylm_krr(),
    ]

def build_gylm_krr():
    return btf.Module(
        tag="gylm_krr",
        transforms=[
            btf.ExtXyzInput(tag="input"),
            btf.GylmAtomic(
                tag="descriptor_atomic",
                args={
                    "normalize": True,
                    "rcut": 7.5,
                    "heavy_only": True,
                    "rcut_width": 0.5,
                    "nmax": 9,
                    "lmax": 7,
                    "sigma": 0.75,
                    "part_sigma": 0.5,
                    "wconstant": False,
                    "wscale": 0.5,
                    "wcentre": 0.5,
                    "ldamp": 0.5,
                    "power": True,
                },
                inputs={"configs": "input.configs"},
            ),
            btf.ReduceTypedMatrix(
                tag="descriptor",
                precompute=True,
                args={
                    "reduce_op": "sum",
                    "normalize": True,
                    "reduce_by_type": False,
                    "types": None,
                    "epsilon": 1e-10,
                },
                inputs={"X": "descriptor_atomic.X", "T": None},
            ),
            btf.KernelDot(tag="kernel", inputs={"X": "descriptor.X"}),
            btf.DoDivideBySize(
                tag="input_norm",
                args={
                    "config_to_size": "lambda c: len(c.getHeavy()[0])",
                    "force": False,
                    "skip_if_not_force": False,
                },
                inputs={"configs": "input.configs", "meta": "input.meta", "y": "input.y"},
            ),
            btf.KernelRidge(
                tag="predictor",
                args={"alpha": None, "power": 4},
                inputs={"K": "kernel.K", "y": "input_norm.y"},
            ),
            btf.UndoDivideBySize(
                tag="output", inputs={"y": "predictor.y", "sizes": "input_norm.sizes"}
            ),
        ],
        hyper=GridHyper(
            Hyper(
                {
                    "predictor.alpha": regularization_range,
                }
            )
        ),
        broadcast={"meta": "input.meta"},
        outputs={"y": "output.y"},
    )

def build_hloops():
    return [
        GridHyper(
            Hyper({"descriptor_atomic.normalize": [False, True]}),
            Hyper({"descriptor_atomic.rcut": [2.0, 2.5, 3.0, 3.5, 4.0, 4.5, 5.5, 6.5, 7.5]}),
            Hyper({"descriptor_atomic.nmax": [6, 7, 8, 9, 10]}),
            Hyper({"descriptor_atomic.lmax": [4, 5, 6, 7]}),
            Hyper({"descriptor_atomic.sigma": [0.25, 0.5, 0.75]}),
            Hyper({"descriptor_atomic.part_sigma": [0.25, 0.5, 0.75]}),
            Hyper({"descriptor_atomic.wconstant": [False, True]}),
            Hyper({"descriptor_atomic.wscale": [0.5, 1.0, 1.5]}),
            Hyper({"descriptor_atomic.wcentre": [0.5, 1.0, 1.5]}),
            Hyper({"descriptor_atomic.ldamp": [0.5, 1.0, 2.0, 4.0]}),
            # Hyper({
            #     "descriptor_atomic.heavy_only": [False, True],
            #     "input_norm.config_to_size": [
            #         "lambda c: len(c)", "lambda c: len(c.getHeavy()[0]"]}),
            Hyper(
                {
                    "descriptor.reduce_op": ["mean", "sum", "sum"],
                    "descriptor.normalize": [False, False, True],
                }
            ),
            Hyper({"predictor.power": [1, 2, 3, 4]}),
        ),
    ]

def evaluate_single(
    dataset,
    model,
    hyper=None,
    split=None,
    hsplit=None,
):
    if split is None:
        split = dict(method="random", n_splits=5, train_fraction=0.9)
    if hsplit is None:
        hsplit = dict(method="random", n_splits=3, train_fraction=0.75)
    log << log.mg << "Model %s" % model.tag << log.endl
    if hyper is not None:
        model.hyperUpdate(hyper, verbose=True)
    stream = model.open(dataset, verbose=True)
    accu = bml.Accumulator(metrics=["mse", "mae", "r2", "rhop"])
    model.precompute(stream, verbose=True)
    for idx, (stream_train, stream_test) in enumerate(stream.split(**split)):
        log << log.back << "  Split %3d" % idx << log.flush
        if idx == 0:
            model.hyperfit(
                stream=stream_train,
                split_args=hsplit,
                accu_args={"metric": dataset["metrics"][0]},
                target="y",
                target_ref="input.y",
                log=log,
            )
        else:
            model.fit(stream_train, verbose=True)
        output_train = model.map(stream_train)
        output_test = model.map(stream_test)
        accu.append("train", output_train["y"], stream_train.resolve("input.y"))
        accu.append("test", output_test["y"], stream_test.resolve("input.y"))
    log << log.endl
    return accu.evaluateAll(log=bml.log, bootstrap=0)

if __name__ == "__main__":
    dataset = bml.load_dataset(
        "data/aqsol_1144.xyz",
        meta={
            "target": "log_sol_mol_l",
            "elements": ["C", "N", "O", "S", "H", "F", "Cl", "Br", "I", "P", "B", "Si"],
            "name": "?",
            "task": "regression",
            "scaling": "non-additive",
            "metrics": ["mse", "mae"],
        },
    )
    n_hyper_settings = 10
    for model, hloop in zip(build_models(), build_hloops()):
        dump = []
        for i in range(n_hyper_settings):
            log << log.mg << "Hyperloop" << i << log.endl
            hargs = hloop.random()
            metrics = evaluate_single(dataset, model, hargs)
            dump.append({"m": metrics, "h": hargs})
        json.dump(dump, open(model.tag + "_metrics_brief.json", "w"), indent=1, sort_keys=True)

max_stars_repo_name: rudolfspetrovs/benchml

max_issues_repo_name: rudolfspetrovs/benchml

max_forks_repo_name: rudolfspetrovs/benchml


text:
import random
import numpy as np
import matplotlib.pyplot as plt

from keras.models import Sequential
from keras.layers import Dense
from keras.wrappers.scikit_learn import KerasRegressor
from sklearn.metrics import mean_squared_error

# Generate a sample dataset from random data:
random.seed(123)
def createDataset(N):
 a,b,c,y = [],[],[],[]
 for i in range(N):
  aa = i/10+random.uniform(-4,3)
  bb = i/30+random.uniform(-4,4)
  cc = i/40+random.uniform(-3,3)-5
  yy = (aa+bb+cc/2)/3
  a.append([aa])
  b.append([bb])
  c.append([cc])
  y.append([yy])
 return np.hstack([a,b,c]), np.array(y)

N = 150
x,y = createDataset(N)
x_ax = range(N)
plt.plot(x_ax, x, 'o', label="original value", markersize=3)
plt.plot(x_ax, y, lw=1.5, color="red", label="y")
plt.legend(['original value'])
plt.show()

ext: py

max_stars_repo_name: olihock/machine-learning

max_issues_repo_name: olihock/machine-learning

max_forks_repo_name: olihock/machine-learning

reason: import numpy


text:
# -*- coding: utf-8 -*-
"""
Created on Wed Mar 28 11:25:43 2019
@author: Taufik Sutanto
taufik@tau-data.id
https://tau-data.id

~~Perjanjian Penggunaan Materi & Codes (PPMC) - License:~~
* Modul Python dan gambar-gambar (images) yang digunakan adalah milik dari berbagai sumber sebagaimana yang telah dicantumkan dalam masing-masing license modul, caption atau watermark.
* Materi & Codes diluar point (1) (i.e. "taudata.py" ini & semua slide ".ipynb)) yang digunakan di pelatihan ini dapat digunakan untuk keperluan akademis dan kegiatan non-komersil lainnya.
* Untuk keperluan diluar point (2), maka dibutuhkan izin tertulis dari Taufik Edy Sutanto (selanjutnya disebut sebagai pengarang).
* Materi & Codes tidak boleh dipublikasikan tanpa izin dari pengarang.
* Materi & codes diberikan "as-is", tanpa warranty. Pengarang tidak bertanggung jawab atas penggunaannya diluar kegiatan resmi yang dilaksanakan pengarang.
* Dengan menggunakan materi dan codes ini berarti pengguna telah menyetujui PPMC ini.
"""

from nltk.tokenize import TweetTokenizer; Tokenizer = TweetTokenizer(reduce_len=True)
from tqdm import tqdm, trange
from wordcloud import WordCloud
from sklearn.feature_extraction.text import CountVectorizer
from textblob import TextBlob
from Sastrawi.StopWordRemover.StopWordRemoverFactory import StopWordRemoverFactory
from Sastrawi.Stemmer.StemmerFactory import StemmerFactory
from bs4 import BeautifulSoup as bs
from sklearn.decomposition import LatentDirichletAllocation as LDA
from bz2 import BZ2File as bz2
from scipy import special
from collections import Counter
import re, networkx as nx, matplotlib.pyplot as plt, operator, numpy as np,community, os, csv#, spacy
#from spacy.lang.id import Indonesian
from twython import Twython, TwythonRateLimitError
import json, pandas as pd, itertools, nltk, time
from nltk.corpus import wordnet as wn
from nltk.stem import PorterStemmer;ps = PorterStemmer()
from itertools import chain
from html import unescape
from nltk import sent_tokenize
from unidecode import unidecode
#from datetime import datetime
#from scipy.sparse import csr_matrix
from bs4.element import Comment
import warnings; warnings.simplefilter('ignore')
"""
Ck = ''
Cs = ''
At = ''
As = ''
"""
def loadCorpus(file='', sep=':', dictionary = True):
    file = open(file, 'r', encoding="utf-8", errors='replace')
    F = file.readlines()
    file.close()
    if dictionary:
        fix = {}
        for f in F:
            k, v = f.split(sep)
            k, v = k.strip(), v.strip()
            fix[k] = v
    else:
        fix = set( (w.strip().replace('\ufeff','') for w in F) )
    return fix

def twitter_html2csv(fData, fHasil):
    urlPattern = re.compile(r'http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+')
    print('Loading Data: ', flush = True)
    Tweets, Username, waktu, replies, retweets, likes, Language, urlStatus =  [], [], [], [], [], [], [], []
    soup = bs(open(fData,encoding='utf-8', errors = 'ignore', mode='r'),'html.parser')
    data = soup.find_all('li', class_= 'stream-item')
    for i,t in tqdm(enumerate(data)):
        T = t.find_all('p',class_='TweetTextSize')[0] # Loading tweet
        Tweets.append(bs(str(T),'html.parser').text)
        U = t.find_all('span',class_='username')
        Username.append(bs(str(U[0]),'html.parser').text)
        T = t.find_all('a',class_='tweet-timestamp')[0]# Loading Time
        waktu.append(bs(str(T),'html.parser').text)
        RP = t.find_all('span',class_='ProfileTweet-actionCountForAria')[0]# Loading reply, retweet & Likes
        replies.append(int((bs(str(RP), "lxml").text.split()[0]).replace('.','').replace(',','')))
        RT = t.find_all('span',class_='ProfileTweet-actionCountForAria')[1]
        RT = int((bs(str(RT), "lxml").text.split()[0]).replace('.','').replace(',',''))
        retweets.append(RT)
        L  = t.find_all('span',class_='ProfileTweet-actionCountForAria')[2]
        likes.append(int((bs(str(L), "lxml").text.split()[0]).replace('.','').replace(',','')))
        try:# Loading Bahasa
            L = t.find_all('span',class_='tweet-language')
            Language.append(bs(str(L[0]), "lxml").text)
        except:
            Language.append('')
        url = str(t.find_all('small',class_='time')[0])
        try:
            url = re.findall(urlPattern,url)[0]
        except:
            try:
                mulai, akhir = url.find('href="/')+len('href="/'), url.find('" title=')
                url = 'https://twitter.com/' + url[mulai:akhir]
            except:
                url = ''
        urlStatus.append(url)
    print('Saving Data to "%s" ' %fHasil, flush = True)
    dfile = open(fHasil, 'w', encoding='utf-8', newline='')
    dfile.write('Time, Username, Tweet, Replies, Retweets, Likes, Language, urlStatus\n')
    with dfile:
        writer = csv.writer(dfile)
        for i,t in enumerate(Tweets):
            writer.writerow([waktu[i],Username[i],t,replies[i],retweets[i],likes[i],Language[i],urlStatus[i]])
    dfile.close()
    print('All Finished', flush = True)

def BoxPlot(Data, Labels=None, dots = True, figName = 'boxPlot.png', Title = '',dpi=600):
    xLabel = 'Akurasi'; yLabel = 'Model'
    #Labels = list(dData.keys())
    #Data = [np.array(dData[d]['scores']) for d in dData.keys()]
    N = len(Data); pos = np.arange(N) + 1
    fig = plt.figure(1, figsize=(9, 6))
    ax1 = fig.add_subplot(111)
    ax1.yaxis.grid(True, linestyle='-', which='major', color='lightgrey', alpha=0.5)
    ax1.set_axisbelow(True)
    ax1.set_title(Title); ax1.set_xlabel(xLabel, size=32); ax1.set_ylabel(yLabel)
    BP = ax1.boxplot(Data,positions=pos,labels=Labels,notch=True, vert=False)
    for box in BP['boxes']:
        box.set( color='#7570b3', linewidth=2)
    if dots:
        for i,dt in zip(pos,Data):
            y = dt
            x = np.random.normal(i, 0.03, size=len(y))
            ax1.plot(y, x, 'b.', alpha=0.30, markersize = 5) # Flip this for vertical BP
    fig.savefig(figName, bbox_inches='tight', pad_inches = 0, dpi=dpi)
    plt.show()

def twitter_connect(Ck, Cs, At, As):
    try:
        twitter = Twython(Ck, Cs, At, As)
        user = twitter.verify_credentials()
        print('Welcome "%s" you are now connected to twitter server' %user['name'])
        return twitter
    except:
        print("Connection failed, please check your API keys or connection")

def getTweets(twitter, topic, N = 100, lan = None):
    Tweets, MAX_ATTEMPTS, count, dBreak, next_max_id = [], 3, 0, False, 0
    i = 0
    while True:
        if(count>=N or dBreak):
            print('\nFinished importing %.0f' %count);break
        if(i == 0):
            if lan:
                results=twitter.search(q=topic, lang=lan, count=100, tweet_mode = 'extended')
            else:
                results=twitter.search(q=topic, count=100, tweet_mode = 'extended')

            Tweets.extend(results['statuses'])
            count += len(results['statuses'])
            if count>N:
                print("\rNbr of Tweets captured: {}".format(N), end="")
                Tweets = Tweets[:N]
                dBreak = True; break
            else:
                print("\rNbr of Tweets captured: {}".format(count), end="")

        else:
            try:
                if lan:
                    results=twitter.search(q=topic,include_entities='true',max_id=next_max_id, lang=lan, count=100, tweet_mode = 'extended')
                else:
                    results=twitter.search(q=topic,include_entities='true',max_id=next_max_id, count=100, tweet_mode = 'extended')

                Tweets.extend(results['statuses'])
                count += len(results['statuses'])
                if count>N:
                    print("\rNbr of Tweets captured: {}".format(N), end="")
                    Tweets = Tweets[:N]
                    dBreak = True; break
                else:
                    print("\rNbr of Tweets captured: {}".format(count), end="")

                try:
                    next_results_url_params=results['search_metadata']['next_results']
                    next_max_id=next_results_url_params.split('max_id=')[1].split('&')[0]
                except:
                    print('\nFinished, no more tweets available for query "%s"' %str(topic), flush = True)
                    dBreak = True; break

            except TwythonRateLimitError:
                print('\nRate Limit reached ... sleeping for 15 Minutes', flush = True)
                for itr in trange(15*60):
                    time.sleep(1)
            except:
                print('\nSomething is not right, retrying ... (attempt = {}/{})'.format(i+1,MAX_ATTEMPTS), flush = True)
        i += 1
    return Tweets

def saveTweets(Tweets,file='Tweets.json', plain = False): #in Json Format
    with open(file, 'w') as f:
        for T in Tweets:
            if plain:
                f.write(T+'\n')
            else:
                try:
                    f.write(json.dumps(T)+'\n')
                except:
                    pass
def tag_visible(element):
    if element.parent.name in ['style', 'script', 'head', 'title', 'meta', '[document]']:
        return False
    if isinstance(element, Comment):
        return False
    return True

def text_from_html(body):
    soup = bs(body, 'html.parser')
    texts = soup.findAll(text=True)
    visible_texts = filter(tag_visible, texts)
    return u" ".join(t.strip() for t in visible_texts)

def loadTweets(file='Tweets.json'):
    f=open(file,encoding='utf-8', errors ='ignore', mode='r');T=f.readlines();f.close()
    for i,t in enumerate(T):
        T[i] = json.loads(t.strip())
    return T

def visualize_wordEmbedding(model):
    from sklearn.manifold import TSNE
    vocab = list(model.wv.vocab)
    X = model[vocab]
    print('projecting to lower dimension for visualization, \nplease wait (Might take awhile for large data) ...')
    tsne = TSNE(n_components=2)
    X_tsne = tsne.fit_transform(X)
    df = pd.DataFrame(X_tsne, index=vocab, columns=['x', 'y'])
    print('Drawing ...')
    fig = plt.figure()
    ax = fig.add_subplot(1, 1, 1)
    ax.scatter(df['x'], df['y'])
    for word, pos in df.iterrows():
        ax.annotate(word, pos)
    plt.show()
    print('Done ... ')

def LoadStopWords(lang='en'):
    L = lang.lower().strip()
    if L == 'en' or L == 'english' or L == 'inggris':
        from spacy.lang.en import English as lemmatizer
        #lemmatizer = spacy.lang.en.English
        lemmatizer = lemmatizer()
        #lemmatizer = spacy.load('en')
        stops =  set([t.strip() for t in LoadDocuments(file = 'data/stopwords_eng.txt')[0]])
    elif L == 'id' or L == 'indonesia' or L=='indonesian':
        from spacy.lang.id import Indonesian
        #lemmatizer = spacy.lang.id.Indonesian
        lemmatizer = Indonesian()
        stops = set([t.strip() for t in LoadDocuments(file = 'data/stopwords_id.txt')[0]])
    else:
        print('Warning, language not recognized. Empty StopWords Given')
        stops = set(); lemmatizer = None
    return stops, lemmatizer

def cleanText(T, fix={}, lemma=None, stops = set(), symbols_remove = True, min_charLen = 2, max_charLen = 15, fixTag= False, fixMix=True):
    # lang & stopS only 2 options : 'en' atau 'id'
    # symbols ASCII atau alnum
    #T = 'kupu-kupu warna biru, abu-abu.'
    pattern = re.compile(r'http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+')
    t = re.sub(pattern,' ',T) #remove urls if any
    pattern = re.compile(r'ftp[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+')
    t = re.sub(pattern,' ',t) #remove urls if any
    t = unescape(t) # html entities fix
    if fixTag:
        t = fixTags(t) # fix abcDef
    t = t.lower().strip() # lowercase
    t = unidecode(t)
    t = ''.join(''.join(s)[:2] for _, s in itertools.groupby(t)) # remove repetition
    t = t.replace('\n', ' ').replace('\r', ' ')
    t = sent_tokenize(t) # sentence segmentation. String to list
    for i, K in enumerate(t):
        if symbols_remove:
            K = re.sub(r'[^.,_a-zA-Z0-9 -\.]',' ',K)
        if lemma:
            listKata = lemma(K)
        else:
            listKata = TextBlob(K).words
        cleanList = []
        for token in listKata:
            if lemma:
                if str(token.text) in fix.keys():
                    token = fix[str(token.text)]
                try:
                    token = token.lemma_
                except:
                    token = lemma(token)[0].lemma_
            else:
                if str(token) in fix.keys():
                    token = fix[str(token)]
            if stops:
                if len(token)>=min_charLen and token not in stops:
                    if fixMix and str(token.replace(' ','').replace('-','a')).isalpha():
                        cleanList.append(token)
                    elif not fixMix:
                        cleanList.append(token)
            else:
                if len(token)>=min_charLen:
                    if fixMix and str(token.replace(' ','').replace('-','a')).isalpha():
                        cleanList.append(token)
                    elif not fixMix:
                        cleanList.append(token)
        t[i] = ' '.join(cleanList)
    return ' '.join(t) # Return kalimat lagi

def crawlFiles(dPath,types=None): # dPath ='C:/Temp/', types = 'pdf'
    if types:
        return [dPath+f for f in os.listdir(dPath) if f.endswith('.'+types)]
    else:
        return [dPath+f for f in os.listdir(dPath)]

def LoadDocuments(dPath=None,types=None, file = None): # types = ['pdf','doc','docx','txt','bz2']
    Files, Docs = [], []
    if types:
        for tipe in types:
            Files += crawlFiles(dPath,tipe)
    if file:
        Files = [file]
    if not types and not file: # get all files regardless of their extensions
        Files += crawlFiles(dPath)
    for f in Files:
        if f[-3:].lower() in ['txt', 'dic','py', 'ipynb']:
            try:
                df=open(f,"r",encoding="utf-8", errors='replace')
                Docs.append(df.readlines());df.close()
            except:
                print('error reading{0}'.format(f))
        elif f[-3:].lower()=='bz2':
            try:
                Docs.append(readBz2(f))
            except:
                print('error reading{0}'.format(f))
        elif f[-3:].lower()=='csv':
            Docs.append(pd.read_csv(f))
        else:
            print('Unsupported format {0}'.format(f))
    if file:
        Docs = Docs[0]
    return Docs, Files

def readBz2(file):
    with bz2(file, "r") as bzData:
        txt = []
        for line in bzData:
            try:
                txt.append(line.strip().decode('utf-8','replace'))
            except:
                pass
    return ' '.join(txt)

def countWords(Doc):
    Full_Tokens  = []
    for line in Doc:
        Full_Tokens.append(TextBlob(line.lower()).words)
    Full_Tokens = [kata for line in Full_Tokens for kata in line] # Flatten List
    Words = Counter(Full_Tokens)
    Words = [(kata,freq) for kata,freq in Words.items()]
    Words.sort(key=lambda tup: tup[1])
    Words = Words[::-1] # Reverse = Descending
    frekuensi = [f[1] for f in Words]
    kata = [f[0] for f in Words]
    return kata, frekuensi

def barChart(kata,frekuensi,N=100):
    y_pos = np.arange(len(kata[:N]))
    plt.bar(y_pos, frekuensi[:N], align='center', alpha=0.5)
    plt.xticks(y_pos, kata)
    plt.ylabel('Frekuensi')
    plt.title('Token')
    plt.show()

def Zipf(frekuensi, par=2, nBins = 100, yScale=0.05):
    count, bins, ignored = plt.hist(np.array(frekuensi), nBins, normed=True)
    plt.title("Zipf plot")
    x = np.arange(1., nBins)
    plt.xlabel("Frequency Rank of Token")
    y = x**(-par) / special.zetac(par)
    plt.ylabel("Absolute Frequency of Token")
    plt.plot(x, y*yScale/max(y), linewidth=2, color='r')
    plt.show()

def WordNet_id(f1 = 'data/wn-ind-def.tab', f2 = 'data/wn-msa-all.tab'):
    w1, wn_id = {}, {}
    df=open(f1,"r",encoding="utf-8", errors='replace')
    d1=df.readlines();df.close()
    df=open(f2,"r",encoding="utf-8", errors='replace')
    d2=df.readlines();df.close(); del df
    for line in d1:
        data = line.split('\t')
        w1[data[0].strip()] = data[-1].strip()
    for line in d2:
        data = line.split('\t')
        kata = data[-1].strip()
        kode = data[0].strip()
        if data[1].strip()=="I":
            if kode in w1.keys():
                if kata in wn_id:
                    wn_id[kata]['def'].append(w1[kode])
                    wn_id[kata]['pos'].append(kode[-1])
                else:
                    wn_id[kata] = {}
                    wn_id[kata]['def'] = [w1[kode]]
                    wn_id[kata]['pos'] = [kode[-1]]
            #else:
            #    wn_id[kata] = {}
            #    wn_id[kata]['def'] = ['']
            #    wn_id[kata]['pos'] = [kode[-1]]
    return wn_id

def loadPos_id(file = 'data/kata_dasar.txt'):
    kata_pos = {}
    df=open(file,"r",encoding="utf-8", errors='replace')
    data=df.readlines();df.close()
    for line in data:
        d = line.split()
        kata = d[0].strip()
        pos = d[-1].strip().replace("(",'').replace(')','')
        kata_pos[kata] = pos
    return kata_pos

def lesk_wsd(sentence, ambiguous_word, pos=None, stem=True, hyperhypo=True):
    # https://en.wikipedia.org/wiki/Lesk_algorithm
    # https://stackoverflow.com/questions/20896278/word-sense-disambiguation-algorithm-in-python
    max_overlaps = 0; lesk_sense = None
    context_sentence = sentence.split()
    for ss in wn.synsets(ambiguous_word):
        #break
        if pos and ss.pos is not pos: # If POS is specified.
            continue
        lesk_dictionary = []
        lesk_dictionary+= ss.definition().replace('(','').replace(')','').split() # Includes definition.
        lesk_dictionary+= ss.lemma_names() # Includes lemma_names.
        # Optional: includes lemma_names of hypernyms and hyponyms.
        if hyperhypo == True:
            lesk_dictionary+= list(chain(*[i.lemma_names() for i in ss.hypernyms()+ss.hyponyms()]))

        if stem == True: # Matching exact words causes sparsity, so lets match stems.
            lesk_dictionary = [ps.stem(i) for i in lesk_dictionary]
            context_sentence = [ps.stem(i) for i in context_sentence]

        overlaps = set(lesk_dictionary).intersection(context_sentence)

        if len(overlaps) > max_overlaps:
            lesk_sense = ss
            max_overlaps = len(overlaps)
    return lesk_sense.name()

def words(text): return re.findall(r'\w+', text.lower())

corpus = 'data/kata_dasar.txt'
WORDS = Counter(words(open(corpus).read()))

def P(word):
    "Probability of `word`."
    N=sum(WORDS.values())
    return WORDS[word] / N

def correction(word):
    "Most probable spelling correction for word."
    return max(candidates(word), key=P)

def candidates(word):
    "Generate possible spelling corrections for word."
    return (known([word]) or known(edits1(word)) or known(edits2(word)) or [word])

def known(words):
    "The subset of `words` that appear in the dictionary of WORDS."
    return set(w for w in words if w in WORDS)

def edits1(word):
    "All edits that are one edit away from `word`."
    letters    = 'abcdefghijklmnopqrstuvwxyz'
    splits     = [(word[:i], word[i:])    for i in range(len(word) + 1)]
    deletes    = [L + R[1:]               for L, R in splits if R]
    transposes = [L + R[1] + R[0] + R[2:] for L, R in splits if len(R)>1]
    replaces   = [L + c + R[1:]           for L, R in splits if R for c in letters]
    inserts    = [L + c + R               for L, R in splits for c in letters]
    return set(deletes + transposes + replaces + inserts)

def edits2(word):
    "All edits that are two edits away from `word`."
    return (e2 for e1 in edits1(word) for e2 in edits1(e1))

def saveTweets_old(Tweets,file='Tweets.json', plain = False): #in "Json" Format or "txt" in plain type
    with open(file, 'w') as f:
        for T in Tweets:
            if plain:
                try:
                    f.write(T['nlp']+'\n')
                except:
                    f.write(T['fullTxt']+'\n')
            else:
                try:
                    f.write(json.dumps(T)+'\n')
                except:
                    pass

def loadTweets_old(file):
    f=open(file,encoding='utf-8', errors ='ignore', mode='r');T=f.readlines();f.close()
    for i,t in enumerate(T):
        T[i] = json.loads(t.strip())
    return T

def strip_non_ascii(string,symbols):
    ''' Returns the string without non ASCII characters''' #isascii = lambda s: len(s) == len(s.encode())
    stripped = (c for c in string if 0 < ord(c) < 127 and c not in symbols)
    return ''.join(stripped)

def adaAngka(s):
    return any(i.isdigit() for i in s)

def fixTags(t):
    getHashtags = re.compile(r"#(\w+)")
    pisahtags = re.compile(r'[A-Z][^A-Z]*')
    tagS = re.findall(getHashtags, t)
    for tag in tagS:
        if len(tag)>0:
            tg = tag[0].upper()+tag[1:]
            proper_words = []
            if adaAngka(tg):
                tag2 = re.split('(\d+)',tg)
                tag2 = [w for w in tag2 if len(w)>0]
                for w in tag2:
                    try:
                        _ = int(w) # error if w not a number
                        proper_words.append(w)
                    except:
                        w = w[0].upper()+w[1:]
                        proper_words = proper_words+re.findall(pisahtags, w)
            else:
                proper_words = re.findall(pisahtags, tg)
            proper_words = ' '.join(proper_words)
            t = t.replace('#'+tag, proper_words)
    return t

def we2vsm(model_we, data_we):
    N = len(data_we)
    L = model_we.vector_size
    vsm_we = np.empty([N, L], dtype=np.float64) # inisialisasi matriks
    for i,d in tqdm(enumerate(data_we)):
        tmp = np.zeros([1, L], dtype=np.float64)
        count = 0
        for t in d:
            try:
                tmp += model_we.wv.__getitem__([t])
                count += 1
            except:
                pass
        if count>0:
            vsm_we[i] = tmp/count
    return vsm_we

def cleanTweets(Tweets):
    factory = StopWordRemoverFactory(); stopwords = set(factory.get_stop_words()+['twitter','rt','pic','com','yg','ga','https'])
    factory = StemmerFactory(); stemmer = factory.create_stemmer()
    for i,tweet in enumerate(tqdm(Tweets)):
        txt = tweet['fullTxt'] # if you want to ignore retweets  ==> if not re.match(r'^RT.*', txt):
        txt = re.sub(r'http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+',' ',txt)# clean urls
        txt = txt.lower() # Lowercase
        txt = Tokenizer.tokenize(txt)
        symbols = set(['@']) # Add more if you want
        txt = [strip_non_ascii(t,symbols) for t in txt] #remove all non ASCII characters
        txt = ' '.join([t for t in txt if len(t)>1])
        Tweets[i]['cleanTxt'] = txt # this is not a good Python practice, only for learning.
        txt = stemmer.stem(txt).split()
        Tweets[i]['nlp'] = ' '.join([t for t in txt if t not in stopwords])
    return Tweets

def translate(txt,language='en'): # txt is a TextBlob object
    try:
        return txt.translate(to=language)
    except:
        return txt

def sentiment_old(Tweets): #need a clean tweets
    print("Calculating Sentiment and Subjectivity Score: ... ")
    T = [translate(TextBlob(tweet['cleanTxt'])) for tweet in tqdm(Tweets)]
    Sen = [tweet.sentiment.polarity for tweet in tqdm(T)]
    Sub = [float(tweet.sentiment.subjectivity) for tweet in tqdm(T)]
    Se, Su = [], []
    for score_se, score_su in zip(Sen,Sub):
        if score_se>0.1:
            Se.append('pos')
        elif score_se<-0.05: #I prefer this
            Se.append('neg')
        else:
            Se.append('net')
        if score_su>0.5:
            Su.append('Subjektif')
        else:
            Su.append('Objektif')
    label_se = ['Positif','Negatif', 'Netral']
    score_se = [len([True for t in Se if t=='pos']),len([True for t in Se if t=='neg']),len([True for t in Se if t=='net'])]
    label_su = ['Subjektif','Objektif']
    score_su = [len([True for t in Su if t=='Subjektif']),len([True for t in Su if t=='Objektif'])]
    PieChart(score_se,label_se); PieChart(score_su,label_su)
    Sen = [(s,t['fullTxt']) for s,t in zip(Sen,Tweets)]
    Sen.sort(key=lambda tup: tup[0])
    Sub = [(s,t['fullTxt']) for s,t in zip(Sub,Tweets)]
    Sub.sort(key=lambda tup: tup[0])
    return (Sen, Sub)

def sentiment(D): #need a clean tweets
    print("Calculating Sentiment and Subjectivity Score: ... ")
    T = [translate(TextBlob(t)) for t in tqdm(D)]
    Sen = [tweet.sentiment.polarity for tweet in tqdm(T)]
    Sub = [float(tweet.sentiment.subjectivity) for tweet in tqdm(T)]
    Se, Su = [], []
    for score_se, score_su in zip(Sen,Sub):
        if score_se>0.0:
            Se.append('pos')
        elif score_se<0.0: #I prefer this
            Se.append('neg')
        else:
            Se.append('net')
        if score_su>0.5:
            Su.append('Subjektif')
        else:
            Su.append('Objektif')
    label_se = ['Positif','Negatif', 'Netral']
    score_se = [len([True for t in Se if t=='pos']),len([True for t in Se if t=='neg']),len([True for t in Se if t=='net'])]
    label_su = ['Subjektif','Objektif']
    score_su = [len([True for t in Su if t=='Subjektif']),len([True for t in Su if t=='Objektif'])]
    PieChart(score_se,label_se); PieChart(score_su,label_su)
    Sen = [(s,t) for s,t in zip(Sen,D)]
    Sen.sort(key=lambda tup: tup[0])
    Sub = [(s,t) for s,t in zip(Sub,D)]
    Sub.sort(key=lambda tup: tup[0])
    return (Sen, Sub)

def printSA(SA, N = 2, emo = 'positif'):
    Sen, Sub = SA
    e = emo.lower().strip()
    if e=='positif' or e=='positive':
        tweets = Sen[-N:]
    elif e=='negatif' or e=='negative':
        tweets = Sen[:N]
    elif e=='netral' or e=='neutral':
        net = [(abs(score),t) for score,t in Sen if abs(score)<0.01]
        net.sort(key=lambda tup: tup[0])
        tweets = net[:N]
    elif e=='subjektif' or e=='subjective':
        tweets = Sub[-N:]
    elif e=='objektif' or e=='objective':
        tweets = Sub[:N]
    else:
        print('Wrong function input parameter = "{0}"'.format(emo)); tweets=[]
    print('"{0}" Tweets = '.format(emo))
    for t in tweets:
        print(t)

def wordClouds(Tweets, file = 'wordCloud.png', plain = False, stopwords=None):
    if plain: # ordinary (large) Text file - String
        txt = Tweets
    else:
        txt = [t['full_text'] for t in Tweets]; txt = ' '.join(txt)
    wc = WordCloud(background_color="white")#, max_font_size=40
    wordcloud = wc.generate(txt)
    plt.figure(num=1, facecolor='w', edgecolor='k') #figsize=(4, 3), dpi=600, #wc.to_file('wordCloud.png')
    plt.imshow(wordcloud, cmap=plt.cm.jet, interpolation='nearest', aspect='auto'); plt.xticks(()); plt.yticks(())
    #plt.savefig('wordCloud.png',bbox_inches='tight', pad_inches = 0.1, dpi=300)
    plt.show()

def PieChart(score,labels):
    fig1 = plt.figure(); fig1.add_subplot(111)
    plt.pie(score, labels=labels, autopct='%1.1f%%', startangle=140)
    plt.axis('equal');plt.show()
    return None

def drawGraph(G, Label, layOut='spring'):
    fig3 = plt.figure(); fig3.add_subplot(111)
    if layOut.lower()=='spring':
        pos = nx.spring_layout(G)
    elif layOut.lower()=='circular':
        pos=nx.circular_layout(G)
    elif layOut.lower()=='random':
        pos = nx.random_layout(G)
    elif layOut.lower()=='shells':
        shells = [G.core_nodes,sorted(G.major_building_routers, key=lambda n: nx.degree(G.topo, n)) + G.distribution_routers + G.server_nodes,G.hosts + G.minor_building_routers]
        pos = nx.shell_layout(G, shells)
    elif layOut.lower()=='spectral':
        pos=nx.spectral_layout(G)
    else:
        print('Graph Type is not available.')
        return
    nx.draw_networkx_nodes(G,pos, alpha=0.2,node_color='blue',node_size=600)
    if Label:
        nx.draw_networkx_labels(G,pos)
    nx.draw_networkx_edges(G,pos,width=4)
    plt.show()

def Graph(Tweets, Label = False, layOut='spring', plain = True): # Need the Tweets Before cleaning
    print("Please wait, building Graph .... ")
    G=nx.Graph()
    if plain:
        users = Tweets[0]
        tweet = Tweets[1]
        for u,t in zip(users, tweet):
            if u not in G.nodes():
                G.add_node(u)
            mentionS =  re.findall("@([a-zA-Z0-9]{1,15})", t)
            for mention in mentionS:
                if "." not in mention: #skipping emails
                    usr = mention.replace("@",'').strip()
                    if usr not in G.nodes():
                        G.add_node(usr)
                    G.add_edge(u,usr)
    else:
        for tweet in tqdm(Tweets):
            if tweet['user']['screen_name'] not in G.nodes():
                G.add_node(tweet['user']['screen_name'])
            mentionS =  re.findall("@([a-zA-Z0-9]{1,15})", tweet['full_text'])
            for mention in mentionS:
                if "." not in mention: #skipping emails
                    usr = mention.replace("@",'').strip()
                    if usr not in G.nodes():
                        G.add_node(usr)
                    G.add_edge(tweet['user']['screen_name'],usr)
    Nn, Ne = G.number_of_nodes(), G.number_of_edges()
    drawGraph(G, Label, layOut)
    print('Finished. There are %d nodes and %d edges in the Graph.' %(Nn,Ne))
    return G

def Centrality(G, N=10, method='katz', outliers=False, Label = True, layOut='shells'):

    if method.lower()=='katz':
        phi = 1.618033988749895 # largest eigenvalue of adj matrix
        ranking = nx.katz_centrality_numpy(G,1/phi)
    elif method.lower() == 'degree':
        ranking = nx.degree_centrality(G)
    elif method.lower() == 'eigen':
        ranking = nx.eigenvector_centrality_numpy(G)
    elif method.lower() =='closeness':
        ranking = nx.closeness_centrality(G)
    elif method.lower() =='betweeness':
        ranking = nx.betweenness_centrality(G)
    elif method.lower() =='harmonic':
        ranking = nx.harmonic_centrality(G)
    elif method.lower() =='percolation':
        ranking = nx.percolation_centrality(G)
    else:
        print('Error, Unsupported Method.'); return None

    important_nodes = sorted(ranking.items(), key=operator.itemgetter(1))[::-1]#[0:Nimportant]
    data = np.array([n[1] for n in important_nodes])
    dnodes = [n[0] for n in important_nodes][:N]
    if outliers:
        m = 1 # 1 standard Deviation CI
        data = data[:N]
        out = len(data[abs(data - np.mean(data)) > m * np.std(data)]) # outlier within m stDev interval
        if out<N:
            dnodes = [n for n in dnodes[:out]]

    print('Influencial Users: {0}'.format(str(dnodes)))
    print('Influencial Users Scores: {0}'.format(str(data[:len(dnodes)])))
    Gt = G.subgraph(dnodes)
    return Gt

def Community(G):
    part = community.best_partition(G)
    values = [part.get(node) for node in G.nodes()]
    mod, k = community.modularity(part,G), len(set(part.values()))
    print("Number of Communities = %d\nNetwork modularity = %.2f" %(k,mod)) # https://en.wikipedia.org/wiki/Modularity_%28networks%29
    fig2 = plt.figure(); fig2.add_subplot(111)
    nx.draw_shell(G, cmap = plt.get_cmap('gist_ncar'), node_color = values, node_size=30, with_labels=False)
    plt.show
    return values

def print_Topics(model, feature_names, Top_Topics, n_top_words):
    for topic_idx, topic in enumerate(model.components_[:Top_Topics]):
        print("Topic #%d:" %(topic_idx+1))
        print(" ".join([feature_names[i]
                        for i in topic.argsort()[:-n_top_words - 1:-1]]))

def getTopics_old(Tweets,n_topics=5, Top_Words=7):
    Txt = [t['nlp'] for t in Tweets] # cleaned: stopwords, stemming
    tf_vectorizer = CountVectorizer(strip_accents = 'unicode', token_pattern = r'\b[a-zA-Z]{3,}\b', max_df = 0.95, min_df = 2)
    dtm_tf = tf_vectorizer.fit_transform(Txt)
    tf_terms = tf_vectorizer.get_feature_names()
    lda_tf = LDA(n_components=n_topics, learning_method='online', random_state=0).fit(dtm_tf)
    vsm_topics = lda_tf.transform(dtm_tf); doc_topic =  [a.argmax()+1 for a in tqdm(vsm_topics)] # topic of docs
    print('In total there are {0} major topics, distributed as follows'.format(len(set(doc_topic))))
    fig4 = plt.figure(); fig4.add_subplot(111)
    plt.hist(np.array(doc_topic), alpha=0.5); plt.show()
    print('Printing top {0} Topics, with top {1} Words:'.format(n_topics, Top_Words))
    print_Topics(lda_tf, tf_terms, n_topics, Top_Words)
    return lda_tf, dtm_tf, tf_vectorizer

def getTopics(Txt,n_topics=5, Top_Words=7):
    tf_vectorizer = CountVectorizer(strip_accents = 'unicode', token_pattern = r'\b[a-zA-Z]{3,}\b', max_df = 0.95, min_df = 2)
    dtm_tf = tf_vectorizer.fit_transform(Txt)
    tf_terms = tf_vectorizer.get_feature_names()
    lda_tf = LDA(n_components=n_topics, learning_method='online', random_state=0).fit(dtm_tf)
    vsm_topics = lda_tf.transform(dtm_tf); doc_topic =  [a.argmax()+1 for a in tqdm(vsm_topics)] # topic of docs
    print('In total there are {0} major topics, distributed as follows'.format(len(set(doc_topic))))
    fig4 = plt.figure(); fig4.add_subplot(111)
    plt.hist(np.array(doc_topic), alpha=0.5); plt.show()
    print('Printing top {0} Topics, with top {1} Words:'.format(n_topics, Top_Words))
    print_Topics(lda_tf, tf_terms, n_topics, Top_Words)
    return lda_tf, dtm_tf, tf_vectorizer

def get_nMax(arr, n):
    indices = arr.ravel().argsort()[-n:]
    indices = (np.unravel_index(i, arr.shape) for i in indices)
    return [(arr[i], i) for i in indices]

def filter_for_tags(tagged, tags=['NN', 'JJ', 'NNP']):
    return [item for item in tagged if item[1] in tags]

def normalize(tagged):
    return [(item[0].replace('.', ''), item[1]) for item in tagged]

def unique_everseen(iterable, key=None):
    "List unique elements, preserving order. Remember all elements ever seen."
    # unique_everseen('AAAABBBCCDAABBB') --> A B C D
    # unique_everseen('ABBCcAD', str.lower) --> A B C D
    seen = set()
    seen_add = seen.add
    if key is None:
        for element in itertools.ifilterfalse(seen.__contains__, iterable):
            seen_add(element)
            yield element
    else:
        for element in iterable:
            k = key(element)
            if k not in seen:
                seen_add(k)
                yield element

def lDistance(firstString, secondString):
    "Function to find the Levenshtein distance between two words/sentences - gotten from http://rosettacode.org/wiki/Levenshtein_distance#Python"
    if len(firstString) > len(secondString):
        firstString, secondString = secondString, firstString
    distances = range(len(firstString) + 1)
    for index2, char2 in enumerate(secondString):
        newDistances = [index2 + 1]
        for index1, char1 in enumerate(firstString):
            if char1 == char2:
                newDistances.append(distances[index1])
            else:
                newDistances.append(1 + min((distances[index1], distances[index1+1], newDistances[-1])))
        distances = newDistances
    return distances[-1]

def buildGraph(nodes):
    "nodes - list of hashables that represents the nodes of the graph"
    gr = nx.Graph() #initialize an undirected graph
    gr.add_nodes_from(nodes)
    nodePairs = list(itertools.combinations(nodes, 2))

    #add edges to the graph (weighted by Levenshtein distance)
    for pair in nodePairs:
        firstString = pair[0]
        secondString = pair[1]
        levDistance = lDistance(firstString, secondString)
        gr.add_edge(firstString, secondString, weight=levDistance)

    return gr

def kataKunci(text):
    #tokenize the text using nltk
    wordTokens = nltk.word_tokenize(text)
    #assign POS tags to the words in the text
    tagged = nltk.pos_tag(wordTokens)
    textlist = [x[0] for x in tagged]

    tagged = filter_for_tags(tagged)
    tagged = normalize(tagged)

    unique_word_set = unique_everseen([x[0] for x in tagged])
    word_set_list = list(unique_word_set)

   #this will be used to determine adjacent words in order to construct keyphrases with two words

    graph = buildGraph(word_set_list)
    #pageRank - initial value of 1.0, error tolerance of 0,0001,
    calculated_page_rank = nx.pagerank(graph, weight='weight')
    #most important words in ascending order of importance
    keyphrases = sorted(calculated_page_rank, key=calculated_page_rank.get, reverse=True)
    #the number of keyphrases returned will be relative to the size of the text (a third of the number of vertices)
    aThird = len(word_set_list) / 3
    keyphrases = keyphrases[0:aThird+1]

    #take keyphrases with multiple words into consideration as done in the paper - if two words are adjacent in the text and are selected as keywords, join them
    #together
    modifiedKeyphrases = set([])
    dealtWith = set([]) #keeps track of individual keywords that have been joined to form a keyphrase
    i = 0
    j = 1
    while j < len(textlist):
        firstWord = textlist[i]
        secondWord = textlist[j]
        if firstWord in keyphrases and secondWord in keyphrases:
            keyphrase = firstWord + ' ' + secondWord
            modifiedKeyphrases.add(keyphrase)
            dealtWith.add(firstWord)
            dealtWith.add(secondWord)
        else:
            if firstWord in keyphrases and firstWord not in dealtWith:
                modifiedKeyphrases.add(firstWord)

            #if this is the last word in the text, and it is a keyword,
            #it definitely has no chance of being a keyphrase at this point
            if j == len(textlist)-1 and secondWord in keyphrases and secondWord not in dealtWith:
                modifiedKeyphrases.add(secondWord)

        i = i + 1
        j = j + 1

    return modifiedKeyphrases

def Rangkum(text,M):
    sent_detector = nltk.data.load('tokenizers/punkt/english.pickle')
    sentenceTokens = sent_detector.tokenize(text.strip())
    graph = buildGraph(sentenceTokens)
    calculated_page_rank = nx.pagerank(graph, weight='weight')
    #most important sentences in ascending order of importance
    sentences = sorted(calculated_page_rank, key=calculated_page_rank.get, reverse=True)
    #return a 100 word summary
    summary = ' '.join(sentences[:M])
    summaryWords = summary.split()
    summaryWords = summaryWords[0:101]
    summary = ' '.join(summaryWords)
    return summary

max_stars_repo_name: bughuntid/ppim

max_issues_repo_name: bughuntid/ppim

max_forks_repo_name: bughuntid/ppim

reason: from scipy


text:
#fuel_cell.py
# by M Vegh, last modified 2/22/2019

#----------------------------------------------------------------------
#   Imports
# ----------------------------------------------------------------------
import sys
sys.path.append('../trunk')
import SUAVE
from SUAVE.Components.Energy.Storages.Batteries import Battery
from SUAVE.Core import Units, Data
from SUAVE.Core import Data
from SUAVE.Methods.Power.Fuel_Cell.Discharge import larminie, setup_larminie, zero_fidelity
from SUAVE.Methods.Power.Fuel_Cell.Sizing import initialize_from_power, initialize_larminie_from_power
import numpy as np
import matplotlib.pyplot as plt

def main():
    numerics                      =Data()
    power                         =np.array([100])

    fuel_cell                     = SUAVE.Components.Energy.Converters.Fuel_Cell()
    fuel_cell.inputs.power_in     = power

    initialize_from_power(fuel_cell, power*2)
    fuel_cell.discharge_model     = zero_fidelity
    conditions                    = Data()
    #build numerics
    numerics.time                 =Data()
    numerics.time.integrate       = np.array([[0, 0],[0, 10]])
    numerics.time.differentiate   = np.array([[0, 0],[0, 1]])

    mdot0       = fuel_cell.energy_calc(conditions, numerics)
    mdot0_truth = 1.0844928369248122e-06
    m0          = fuel_cell.mass_properties.mass
    m0_truth    = 0.09615384615384616
    err_mdot0   = (mdot0 - mdot0_truth)/mdot0_truth
    err_m0      = (m0 - m0_truth)/m0_truth

    #try another discharge model
    fuel_cell.discharge_model    = larminie
    #populate fuel cell with required values for this discharge method
    setup_larminie(fuel_cell)
    initialize_larminie_from_power(fuel_cell,power)
    mdot1          = fuel_cell.energy_calc(conditions, numerics)
    mdot1_truth    = 1.40641687e-06
    err_mdot1      = (mdot1 - mdot1_truth)/mdot1_truth

    err       = Data()
    err.fuel_cell_mass_error          = err_m0
    err.fuel_cell_fidelity_zero_error = err_mdot0
    err.fuel_cell_larminie_error      = err_mdot1
    for k,v in list(err.items()):
        assert(np.abs(v)<1E-6)
    print(err)

if __name__ == '__main__':
    main()


text:
import sys
import os

from datetime import datetime

import re

import numpy as np
import pandas as pd

import sqlite3

start = datetime.now()

database = sqlite3.connect('files/sql.db')
cursor = database.cursor()

def shorten_zip(zipcode):
    #check for candian
    if re.search('[A-Za-z]', zipcode):
        return zipcode
    else:
        return zipcode[:5]

def create_table():
    cursor.execute("DROP TABLE IF EXISTS real_estate_data")
    cursor.execute("CREATE TABLE real_estate_data(state text, city text, zip text, address text, property_type text, beds text, baths text, sqft text, price int, posted_at text)")

columns_list = ['state', 'city', 'zip', 'address', 'property_type', 'beds', 'baths', 'sqft', 'price', 'posted_at']

data_iterator = pd.read_csv("files/renthub_data.csv",
    iterator = True, chunksize = 10**6,
    usecols = columns_list,
    parse_dates = ['posted_at'],
    dtype = {
        'state': str,
        'city': str,
        'zip': str,
        'address': str,
        'property_type': str,
        'beds': int,
        'baths': float,
        'sqft': str,
        'price': str
    },
)

create_table()

for current_frame in data_iterator:
    current_frame.dropna(subset=['state', 'city', 'zip', 'address'], inplace=True)
    current_frame['price'] = current_frame['price'].replace(r'[\$\,]', '').astype(int).round()
    current_frame['zip'] = current_frame['zip'].apply(shorten_zip)
    for row in current_frame.itertuples():
        row_dict = row._asdict()
        sql_row = [str(row_dict[field]) for field in columns_list]
        cursor.execute("INSERT INTO real_estate_data VALUES(?,?,?,?,?,?,?,?,?,?);",sql_row)

database.commit()

print("Program run took: " + str(datetime.now() - start))

max_stars_repo_name: mendelthegeek/HiGeorge-real-estate

max_issues_repo_name: mendelthegeek/HiGeorge-real-estate

max_forks_repo_name: mendelthegeek/HiGeorge-real-estate


text:
"""Learning script for single agent problems.

Agents are based on `stable_baselines3`'s implementation of A2C, PPO SAC, TD3, DDPG.

Example
-------
To run the script, type in a terminal:

    $ python singleagent.py --env <env> --algo <alg> --obs <ObservationType> --act <ActionType> --cpu <cpu_num>

Notes
-----
Use:

    $ tensorboard --logdir ./results/save-<env>-<algo>-<obs>-<act>-<time-date>/tb/

To check the tensorboard results at:

    http://localhost:6006/

"""
import sys

sys.path.append('../../')

import os
import time
from datetime import datetime
from sys import platform
import argparse
import subprocess
import numpy as np
import gym
import torch
from stable_baselines3.common.env_checker import check_env
from stable_baselines3.common.cmd_util import \
    make_vec_env  # Module cmd_util will be renamed to env_util https://github.com/DLR-RM/stable-baselines3/pull/197
from stable_baselines3.common.utils import set_random_seed
from stable_baselines3 import A2C, PPO, SAC, TD3, DDPG
from stable_baselines3.common.policies import ActorCriticPolicy as a2cppoMlpPolicy
from stable_baselines3.sac.policies import SACPolicy as sacMlpPolicy
from stable_baselines3.td3 import MlpPolicy as td3ddpgMlpPolicy
from stable_baselines3.common.callbacks import CheckpointCallback, EvalCallback, StopTrainingOnRewardThreshold

from gym_pybullet_drones.envs.single_agent_rl.MoveAviary import MoveAviary
from gym_pybullet_drones.envs.single_agent_rl.HoverAviary import HoverAviary
from gym_pybullet_drones.envs.single_agent_rl.BaseSingleAgentAviary import ActionType, ObservationType

import shared_constants

EPISODE_REWARD_THRESHOLD = -0  # Upperbound: rewards are always negative, but non-zero
"""float: Reward threshold to halt the script."""

def check_parser(ARGS):
    #### Warning ###############################################
    if ARGS.act == ActionType.ONE_D_RPM or ARGS.act == ActionType.ONE_D_DYN or ARGS.act == ActionType.ONE_D_PID:
        print("\n\n\n[WARNING] Simplified 1D problem for debugging purposes\n\n\n")
        #### Errors ################################################
        if not ARGS.env in ['hover']:
            print("[ERROR] 1D action space is only compatible with HoverAviary")
            exit()
    if ARGS.algo in ['sac', 'td3', 'ddpg'] and ARGS.cpu != 1:
        print("[ERROR] The selected algorithm does not support multiple environments")
        exit()

if __name__ == "__main__":

    #### Define and parse (optional) arguments for the script ##
    parser = argparse.ArgumentParser(description='Single agent reinforcement learning experiments script')
    parser.add_argument('--env', default='move', type=str, choices=['move', 'hover'], help='Task (default: hover)',
                        metavar='')
    parser.add_argument('--algo', default='ppo', type=str, choices=['a2c', 'ppo', 'sac', 'td3', 'ddpg'],
                        help='RL agent (default: ppo)', metavar='')
    parser.add_argument('--obs', default='kin', type=ObservationType, help='Observation space (default: kin)',
                        metavar='')
    parser.add_argument('--act', default='rpm', type=ActionType, help='Action space (default: one_d_rpm)', metavar='')
    parser.add_argument('--cpu', default='1', type=int, help='Number of training environments (default: 1)', metavar='')
    parser.add_argument('--resume', default='results/save-move-ppo-kin-rpm-03.01.2022_19.37.27/success_model.zip',
                        type=str, help='Resume Training from file path', metavar='')

    ARGS = parser.parse_args()
    check_parser(ARGS)

    #### Save directory ####

    filename = os.path.dirname(os.path.abspath(
        __file__)) + '/results/save-' + ARGS.env + '-' + ARGS.algo + '-' + ARGS.obs.value + '-' + ARGS.act.value + '-' + datetime.now().strftime(
        "%m.%d.%Y_%H.%M.%S")
    if not os.path.exists(filename):
        os.makedirs(filename + '/')

    env_name = ARGS.env + "-aviary-v0"

    if env_name == "move-aviary-v0":
        train_env = MoveAviary(initial_xyzs=np.array([[0, 0, 1]]),
                               aggregate_phy_steps=shared_constants.AGGR_PHY_STEPS,
                               obs=ARGS.obs,
                               act=ARGS.act)
    if env_name == "hover-aviary-v0":
        train_env = HoverAviary(initial_xyzs=np.array([[0, 0, 1]]),
                                aggregate_phy_steps=shared_constants.AGGR_PHY_STEPS,
                                obs=ARGS.obs,
                                act=ARGS.act)

    print("[INFO] Action space:", train_env.action_space)
    print("[INFO] Observation space:", train_env.observation_space)

    #### On-policy algorithms ##################################
    onpolicy_kwargs = dict(activation_fn=torch.nn.ReLU,
                           net_arch=[128, 128, dict(vf=[256], pi=[256])]
                           # net_arch=[128, 128, dict(vf=[256], pi=[128, 256])]
                           )
    if ARGS.algo == 'a2c':
        model = A2C(a2cppoMlpPolicy,
                    train_env,
                    policy_kwargs=onpolicy_kwargs,
                    tensorboard_log=filename + '/tb/',
                    verbose=1,
                    seed=0
                    )
    if ARGS.algo == 'ppo':
        model = PPO(a2cppoMlpPolicy,
                    train_env,
                    policy_kwargs=onpolicy_kwargs,
                    tensorboard_log=filename + '/tb/',
                    verbose=1,
                    seed=0
                    )

    #### Off-policy algorithms #################################
    offpolicy_kwargs = dict(activation_fn=torch.nn.ReLU,
                            net_arch=[128, 128, 128, 256]
                            )  # or None # or dict(net_arch=dict(qf=[256, 128, 64, 32], pi=[256, 128, 64, 32]))
    if ARGS.algo == 'sac':
        model = SAC(sacMlpPolicy,
                    train_env,
                    policy_kwargs=offpolicy_kwargs,
                    tensorboard_log=filename + '/tb/',
                    verbose=1
                    )
    if ARGS.algo == 'td3':
        model = TD3(td3ddpgMlpPolicy,
                    train_env,
                    policy_kwargs=offpolicy_kwargs,
                    tensorboard_log=filename + '/tb/',
                    verbose=1,
                    )
    if ARGS.algo == 'ddpg':
        model = DDPG(td3ddpgMlpPolicy,
                     train_env,
                     policy_kwargs=offpolicy_kwargs,
                     tensorboard_log=filename + '/tb/',
                     verbose=1
                     )

    #### Create eveluation environment #########################
    if ARGS.obs == ObservationType.KIN:
        eval_env = gym.make(env_name,
                            aggregate_phy_steps=shared_constants.AGGR_PHY_STEPS,
                            obs=ARGS.obs,
                            act=ARGS.act
                            )

    #### Train the model #######################################
    callback_on_best = StopTrainingOnRewardThreshold(reward_threshold=EPISODE_REWARD_THRESHOLD,
                                                     verbose=1
                                                     )
    eval_callback = EvalCallback(eval_env,
                                 callback_on_new_best=callback_on_best,
                                 verbose=1,
                                 best_model_save_path=filename + '/',
                                 log_path=filename + '/',
                                 eval_freq=int(2000 / ARGS.cpu),
                                 deterministic=True,
                                 render=False
                                 )

    # RESUMES PREVIOUSLY TRAINED MODEL
    if ARGS.resume:
        path_prev_train = ARGS.resume
        if 'ppo' in path_prev_train:
            model = PPO.load(path_prev_train)
        elif 'a2c' in path_prev_train:
            model = A2C.load(path_prev_train)
        elif 'ddpg' in path_prev_train:
            model = DDPG.load(path_prev_train)
        elif 'td3' in path_prev_train:
            model = TD3.load(path_prev_train)
        elif 'sac' in path_prev_train:
            model = SAC.load(path_prev_train)
        model.set_env(eval_env)

    model.learn(total_timesteps=600000,  # int(1e12),
                callback=eval_callback,
                log_interval=100,
                )

    #### Save the model ########################################
    model.save(filename + '/success_model.zip')
    print(filename)

max_stars_repo_name: QuadCtrl/quad-ctrl

max_issues_repo_name: QuadCtrl/quad-ctrl

max_forks_repo_name: QuadCtrl/quad-ctrl


text:
'''
LastEditTime: 2022-01-15 08:26:24
Description: Sampling functions
Date: 2022-01-07 17:59:23
Author: Aiden Li
LastEditors: Aiden Li (i@aidenli.net)
'''
from os import spawnlpe
from numpy.random.mtrand import sample
import torch
import numpy as np
from torch.utils.tensorboard.writer import SummaryWriter

# Debug only
import matplotlib.pyplot as plt
# END Debug only

plot_samples = 5

def its_from_weights(bounds, weights, num_points_fine):
    """Inverse Transform Sampling from piecewise-constant pdf

    Args:
        bounds ([type]): batch_size x (num_points + 1), boundaries of weights
        weights ([type]): batch_size x num_points
        num_point_fine ([type]): num of points to be additionally sampled
    """
    batch_size = bounds.shape[0]

    # Turning into PCPDF
    probabilities = weights / (weights.sum(dim=-1).unsqueeze(-1) + 1e-5)
    cumulative = torch.cumsum(probabilities, dim=-1)

    uni = torch.rand([batch_size, num_points_fine], device=bounds.device)

    positions = torch.searchsorted(cumulative, uni)

    lo = torch.gather(input=bounds, index=positions, dim=-1)        # batch_size x num_points_fine
    hi = torch.gather(input=bounds, index=torch.minimum(positions + 1, torch.ones_like(positions) * (bounds.shape[-1] - 1)), dim=-1)    # batch_size x num_points_fine

    t_rand = torch.rand([batch_size, num_points_fine], device=bounds.device)

    sampled = lo + (hi - lo) * t_rand

    # bd = bounds[0].detach().cpu().numpy()
    # sm = sampled[0].detach().cpu().numpy()

    # Visualize:
    # - PDF
    # global plot_samples
    # if plot_samples > 0:
    #     if cumulative[0, -1] > 0.9:  # is 1
    #         plot_samples -= 1
    #         plt.figure()
    #         plt.scatter(0.5 * (bd[:-1] + bd[1:]), probabilities[0].detach().cpu().numpy(), color='blue', alpha=0.5)
    #         plt.scatter(0.5 * (bd[:-1] + bd[1:]), cumulative[0].detach().cpu().numpy(), color='red', alpha=0.5)
    #         plt.scatter(sm, np.zeros_like(sm), color='black', marker='x')
    #         plt.savefig(f"Hierarcically Sampling { 5 - plot_samples }.jpg")

    return sampled

reason: import numpy,from numpy


text:
# -*- coding: utf-8 -*-
"""
Created on Sun May 29 00:49:35 2016

@author: hossam
"""

#% ======================================================== %
#% Files of the Matlab programs included in the book:       %
#% Xin-She Yang, Nature-Inspired Metaheuristic Algorithms,  %
#% Second Edition, Luniver Press, (2010).   www.luniver.com %
#% ======================================================== %
#
#% -------------------------------------------------------- %
#% Firefly Algorithm for constrained optimization using     %
#% for the design of a spring (benchmark)                   %
#% by Xin-She Yang (Cambridge University) Copyright @2009   %
#% -------------------------------------------------------- %

import numpy
import math
import time
from solution import solution

def alpha_new(alpha,NGen):
    #% alpha_n=alpha_0(1-delta)^NGen=10^(-4);
    #% alpha_0=0.9
    delta=1-(10**(-4)/0.9)**(1/NGen);
    alpha=(1-delta)*alpha
    return alpha

def FFA(objf,lb,ub,dim,n,MaxGeneration):

    #General parameters

    #n=50 #number of fireflies
    #dim=30 #dim
    #lb=-50
    #ub=50
    #MaxGeneration=500

    #FFA parameters
    alpha=0.5  # Randomness 0--1 (highly random)
    betamin=0.20  # minimum value of beta
    gamma=1   # Absorption coefficient

    zn=numpy.ones(n)
    zn.fill(float("inf"))

    #ns(i,:)=Lb+(Ub-Lb).*rand(1,d);
    ns=numpy.random.uniform(0,1,(n,dim)) *(ub-lb)+lb
    Lightn=numpy.ones(n)
    Lightn.fill(float("inf"))

    #[ns,Lightn]=init_ffa(n,d,Lb,Ub,u0)

    convergence=[]
    s=solution()

    print("CS is optimizing  \""+objf.__name__+"\"")

    timerStart=time.time()
    s.startTime=time.strftime("%Y-%m-%d-%H-%M-%S")

    # Main loop
    for k in range (0,MaxGeneration):     # start iterations

        #% This line of reducing alpha is optional
        alpha=alpha_new(alpha,MaxGeneration);

        #% Evaluate new solutions (for all n fireflies)
        for i in range(0,n):
            zn[i]=objf(ns[i,:])
            Lightn[i]=zn[i]

        # Ranking fireflies by their light intensity/objectives

        Lightn=numpy.sort(zn)
        Index=numpy.argsort(zn)
        ns=ns[Index,:]

        #Find the current best
        nso=ns
        Lighto=Lightn
        nbest=ns[0,:]
        Lightbest=Lightn[0]

        #% For output only
        fbest=Lightbest;

        #% Move all fireflies to the better locations
    #    [ns]=ffa_move(n,d,ns,Lightn,nso,Lighto,nbest,...
    #          Lightbest,alpha,betamin,gamma,Lb,Ub);
        scale=numpy.ones(dim)*abs(ub-lb)
        for i in range (0,n):
            # The attractiveness parameter beta=exp(-gamma*r)
            for j in range(0,n):
                r=numpy.sqrt(numpy.sum((ns[i,:]-ns[j,:])**2));
                #r=1
                # Update moves
                if Lightn[i]>Lighto[j]: # Brighter and more attractive
                   beta0=1
                   beta=(beta0-betamin)*math.exp(-gamma*r**2)+betamin
                   tmpf=alpha*(numpy.random.rand(dim)-0.5)*scale
                   ns[i,:]=ns[i,:]*(1-beta)+nso[j,:]*beta+tmpf

        #ns=numpy.clip(ns, lb, ub)

        convergence.append(fbest)

        IterationNumber=k
        BestQuality=fbest

        if (k%1==0):
               print(['At iteration '+ str(k)+ ' the best fitness is '+ str(BestQuality)])
    #
       ####################### End main loop
    timerEnd=time.time()
    s.endTime=time.strftime("%Y-%m-%d-%H-%M-%S")
    s.executionTime=timerEnd-timerStart
    s.convergence=convergence
    s.optimizer="FFA"
    s.objfname=objf.__name__

    return s

max_stars_repo_name: karthiksekaran/swarm-algorithms

max_issues_repo_name: karthiksekaran/swarm-algorithms

max_forks_repo_name: karthiksekaran/swarm-algorithms


text:
from numpy import array, concatenate, zeros

from .. import differentiate
from .affine_dynamics import AffineDynamics

class LearnedAffineDynamics(AffineDynamics):
    def __init__(self, affine_dynamics, res_aff_model):
        self.dynamics = affine_dynamics
        self.res_model = res_aff_model

    def process_drift(self, x, t):
        return concatenate([x, array([t])])

    def process_act(self, x, t):
        return concatenate([x, array([t])])

    def eval(self, x, t):
        return self.dynamics.eval(x, t)

    def drift(self, x, t):
        return self.dynamics.drift(x, t) + self.res_model.eval_drift(self.process_drift(x, t))

    def act(self, x, t):
        return self.dynamics.act(x, t) + self.res_model.eval_act(self.process_act(x, t))

    def process_episode(self, xs, us, ts, window=3):
        alpha = 10
        comp_safety = lambda r: alpha * r

        half_window = (window - 1) // 2
        xs = xs[:len(us)]
        ts = ts[:len(us)]

        drift_inputs = array([self.process_drift(x, t) for x, t in zip(xs, ts)])
        act_inputs = array([self.process_act(x, t) for x, t in zip(xs, ts)])

        reps = array([self.dynamics.eval(x, t) for x, t in zip(xs, ts)])
        rep_dots = differentiate(reps, ts)

        rep_dot_noms = array([self.dynamics.eval_dot(x, u, t) for x, u, t in zip(xs, us, ts)])

        apreds = zeros(ts.size,)
        bpreds = zeros(ts.size,)
        apredsvar = zeros(ts.size,)
        bpredsvar = zeros(ts.size,)
        respreds = zeros(ts.size,)
        #j = 0
        #if self.res_model is not None:
        #  for x,u,t in zip(xs,us,ts):
        #    meanb,varb,varab = self.drift(x,t)
        #    meana,vara = self.act(x,t)
        #    meanb = meanb - comp_safety(self.eval(x,t))
        #    apreds[j] = meana - self.dynamics.act(x, t)
        #    bpreds[j] = meanb - self.dynamics.drift(x, t)
        #    apredsvar[j] = vara
        #    bpredsvar[j] = varb
        #    respreds[j] = meana*u+meanb
        #    j = j+1

        drift_inputs = drift_inputs[half_window:-half_window]
        act_inputs = act_inputs[half_window:-half_window]
        rep_dot_noms = rep_dot_noms[half_window:-half_window]

        #apreds = apreds[half_window:-half_window]
        #apredsvar = apredsvar[half_window:-half_window]
        #bpreds = bpreds[half_window:-half_window]
        #respreds = respreds[half_window:-half_window]
        #bpredsvar = bpredsvar[half_window:-half_window]

        residuals = rep_dots - rep_dot_noms

        return drift_inputs, act_inputs, us, residuals

    def init_data(self, d_drift_in, d_act_in, m, d_out):
        return [zeros((0, d_drift_in)), zeros((0, d_act_in)), zeros((0, m)), zeros((0, d_out))]

    def fit(self, data, batch_size=1, num_epochs=1, validation_split=0):
        drift_inputs, act_inputs, us, residuals = data
        self.res_model.fit(drift_inputs, act_inputs, us, residuals, batch_size, num_epochs, validation_split)

max_stars_repo_name: athindran/ProBF

max_issues_repo_name: athindran/ProBF

max_forks_repo_name: athindran/ProBF

reason: from numpy


text:
import numpy as np
import matplotlib.pyplot as plt

if __name__ == '__main__':
    N = [100, 1000, 10000]
    t1 = np.array([977.13, 7633.74, 73122.74])  # N_STEPS = 10000
    t2 = np.array([1865.90, 15072.08, 144255.00])  # N_STEPS = 20000
    t1 /= 12  # number of realizations / parameters
    t2 /= 12
    t1 /= 60  # in minutes
    t2 /= 60

    plt.grid(alpha=0.1)
    plt.plot(N, t1, 'o-', label='N_STEPS=10000', linewidth=3, color='red')
    plt.plot(N, t2, 'o-', label='N_STEPS=20000', linewidth=3, color='blue')
    plt.ylabel('Time [min]')
    plt.xlabel('Network size')
    plt.xscale('log')
    plt.yscale('log')
    plt.legend()
    # plt.savefig('measure_time_simulation_results.pdf', bbox_inches='tight')
    plt.show()


text:
import networkx as nx
import matplotlib.pyplot as plt

G = nx.read_multiline_adjlist('graph.multiline_adjlist')
nx.draw_networkx(G,arrows=True,node_size = 180,node_shape = "o")
# nx.draw_spectral(G,arrows=True,node_size = 200,node_shape = "8",node_color = "red")

# nx.write_gml(G,"test.gml")
plt.show()

reason: import networkx


text:
import json
import numpy as np
from sklearn.model_selection import train_test_split
import tensorflow.keras as keras
import matplotlib.pyplot as plt

DATASET_PATH = "D:\Datasets\GTZAN Dataset - Music Genre Classification\data.json"

def load_data(dataset_path):
    with open(dataset_path, "r") as fp:
        data = json.load(fp)

        # convert list into numpy arrays
        inputs = np.array(data["mfcc"])
        targets = np.array(data["labels"])

        return inputs, targets

def plot_history(history):
    fig, axs = plt.subplots(2)

    # create accuracy subplot

    axs[0].plot(history.history["accuracy"], label="train accuracy")
    axs[0].plot(history.history["val_accuracy"], label="test accuracy")
    axs[0].set_ylabel("Accuracy")
    axs[0].legend(loc="lower right")
    axs[0].set_title("Accuracy eval")

    # create error subplot
    axs[1].plot(history.history["loss"], label="train error")
    axs[1].plot(history.history["val_loss"], label="test error")
    axs[1].set_ylabel("Error")
    axs[1].set_xlabel("Epoch")
    axs[1].legend(loc="upper right")
    axs[1].set_title("Error eval")

    plt.show()

if __name__ == "__main__":
    # load data
    inputs, targets = load_data(DATASET_PATH)

    # split data into train and test sets
    inputs_train, inputs_test, targets_train, targets_test = train_test_split(inputs, targets,
                                                                              test_size=0.3)

    # build the network archuitecture
    model = keras.Sequential([
        # input layer
        keras.layers.Flatten(input_shape=(inputs.shape[1], inputs.shape[2])),  # hop_lenght

        # 1st hidden layer
        keras.layers.Dense(512, activation="relu", kernel_regularizer=keras.regularizers.l2(0.001)),  # Rectified Linear Unit
        keras.layers.Dropout(0.3),

        # 2nd hidden layer
        keras.layers.Dense(6, ac25tivation="relu", kernel_regularizer=keras.regularizers.l2(0.001)),  # 512/256/65 are the number of neurons
        keras.layers.Dropout(0.3),

        # 3rd hidden layer
        keras.layers.Dense(64, activation="relu", kernel_regularizer=keras.regularizers.l2(0.001)),
        keras.layers.Dropout(0.3),

        # output layer
        keras.layers.Dense(10, activation="softmax")  # Is 10 neurons because we have 10 musical genres
    ])

    # compile network
    optimizer = keras.optimizers.Adam(learning_rate=0.0001)  # stochastic gradient descent
    model.compile(optimizer=optimizer,
                  loss="sparse_categorical_crossentropy",
                  metrics=["accuracy"])
    model.summary()

    # train network
    history = model.fit(inputs_train, targets_train,
                        validation_data=(inputs_test, targets_test),
                        epochs=100,
                        batch_size=32)  # see in slide  # for use the whole dataset, configure the batch_size=1

    # plot accuracy and error over the epochs
    plot_history(history)

max_stars_repo_path: Deep Learning (for Audio) with Python/14- SOLVING OVERFITTING in neural networks/genre_classifier.py