text:
# -*- code:utf-8 -*-

"""
application of nltk in classification
"""

import nltk
from nltk.corpus import stopwords
from nltk.stem import WordNetLemmatizer
from gensim import corpora, models, similarities
import csv
from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer
import numpy as np
from sklearn.naive_bayes import MultinomialNB
from sklearn.metrics import confusion_matrix, classification_report
from sklearn import tree
from sklearn.linear_model import SGDClassifier
from sklearn.svm import LinearSVC
from sklearn.ensemble import RandomForestClassifier
from sklearn.linear_model import LogisticRegression
from sklearn.cluster import KMeans, MiniBatchKMeans
from collections import defaultdict
from itertools import chain
from operator import itemgetter
import re

# 对文本进行预处理
def preprocessing(text):

    tokens = [ word for sentence in nltk.sent_tokenize(text) for word in nltk.word_tokenize(sentence)]
    stop  = stopwords.words('english')
    # print(stop)

    # remove the stopwords
    tokens = [token for token in tokens if token not in stop]
    # removing words whose length are less than 3
    tokens = [word for word in tokens if len(word) >= 3]
    tokens = [word.lower() for word in tokens]  # lowering alphabat

    # 词形还原
    lemmatizer = WordNetLemmatizer()
    tokens = [lemmatizer.lemmatize(word) for word in tokens]

    processed_text = ' '.join(tokens)

    return processed_text

# open txt file, extract data and label
sms = open('./Machine-Learning-with-R-datasets-master/SMSSpamCollection.txt', encoding='utf8')
sms_datas = []
sms_labels = []
csv_reader = csv.reader(sms, delimiter='\t')
for line in csv_reader:
    sms_labels.append(line[0])
    sms_datas.append(preprocessing(line[1]))
sms.close()

# sampling  for  train and test
train_set_size = int(round(len(sms_labels)*0.70))

# print('The training set size for this classifier is ' + str(train_set_size) + '\n')
# x_train_set = np.array([''.join(el) for el in sms_datas[0: train_set_size]])
# y_train_set = np.array([el for el in sms_labels[0: train_set_size]])
# x_test_set = np.array([''.join(el) for el in sms_datas[train_set_size+1:len(sms_datas)]])
# y_test_set = np.array([el for el in sms_labels[train_set_size+1:len(sms_labels)]])

print('train set size :' + str(train_set_size) + '\n')

# should consider ramdoning the data later
x_train_set = np.array(sms_datas[0:train_set_size])
y_train_set = np.array(sms_labels[0:train_set_size])

x_test_set = np.array(sms_datas[train_set_size:-1])
y_test_set = np.array(sms_labels[train_set_size:-1])

# print(x_train_set)
# print(y_train_set)

# extract features
sms_exp = []
print('sms_datas length :' + str(len(sms_datas)))  # 5572
for line in sms_datas:
    sms_exp.append(preprocessing(line))
print('sms_exp length :' + str(len(sms_exp)))  # 5572

vectorizer = CountVectorizer(min_df=1, encoding='utf-8')
# return csr matrix of shape = (len(sms_exp), tokens)
X_exp = vectorizer.fit_transform(sms_exp)
# print(X_exp)
# print('||'.join(vectorizer.get_feature_names()))
# print(X_exp.toarray())

vectorizer_1 = TfidfVectorizer(min_df=2, ngram_range=(1, 2), stop_words='english',
                               strip_accents='unicode', norm='l2')
X_train_set = vectorizer_1.fit_transform(x_train_set)
X_test_set = vectorizer_1.transform(x_test_set)

# print(X_train_set)
# print(X_test_set)

# --------------------  Naive Bayes CLS  -------------------------------
print('# --------------------  Naive Bayes CLS  -------------------------------')

classifier_naive_bayes = MultinomialNB().fit(X_train_set, y_train_set)
y_bayes_predict = classifier_naive_bayes.predict(X_test_set)

# print(y_bayes_predict)
print('\n confusion_matrix \n')
confusion_M = confusion_matrix(y_test_set,y_bayes_predict)
# print(y_test_set)
print(confusion_M)

print('\n Here is the classification report:')
print(classification_report(y_test_set,y_bayes_predict))

feature_names = vectorizer_1.get_feature_names()

# print(classifier_naive_bayes.coef_)
# print(classifier_naive_bayes.intercept_)
'''
[[-7.98987527 -8.22274866 -7.24524555 ..., -8.51681462 -8.73944109
  -8.69659372]]
'''
# coefs_with_fns = sorted(zip(classifier_naive_bayes.coef_[0], feature_names))
# _n = 10
# top = zip(coefs_with_fns[0: _n], coefs_with_fns[-_n:])
# for (coe1, fea1), (coe2, fea2) in top:
#     print('\t%.4f\t%-15s\t%.4f\t%-15s' %(coe1, fea1, coe2, fea2))

# --------------------  decision tree CLS  -------------------------------
print('\n--------------------  decision tree CLS  -------------------------------\n')
clf_tree = tree.DecisionTreeClassifier().fit(X_train_set, y_train_set)
y_tree_prediction = clf_tree.predict(X_test_set)

print('\n confusion_matrix \n')
confusion_M_tree = confusion_matrix(y_test_set,y_tree_prediction)
print(confusion_M_tree)

# print(y_tree_prediction)
print('\n Here is the classification report:')
print(classification_report(y_test_set, y_tree_prediction))

# --------------------   sgd CLS -------------------------------
print('\n--------------------  sgd CLS  -------------------------------\n')
clf_sgd = SGDClassifier().fit(X_train_set, y_train_set)
y_sgd_prediction = clf_sgd.predict(X_test_set)
# print(y_sgd_prediction)

print('\n confusion_matrix \n')
confusion_M_sgd = confusion_matrix(y_test_set,y_sgd_prediction)
print(confusion_M_sgd)

print('\n Here is the classification report:')
print(classification_report(y_test_set, y_sgd_prediction))

# --------------------  SVM  -------------------------------
print('\n--------------------  SVM CLS  -------------------------------\n')

cls_svm = LinearSVC().fit(X_train_set, y_train_set)
y_svm_prediction = cls_svm.predict(X_test_set)

print('\n confusion_matrix \n')
confusion_M_svm = confusion_matrix(y_test_set,y_svm_prediction)
print(confusion_M_svm)

print('\n Here is the classification report:')
print(classification_report(y_test_set, y_svm_prediction))

# --------------------  Random Forest -------------------------------
print('\n--------------------  Random Forest CLS  -------------------------------\n')

cls_rf = RandomForestClassifier().fit(X_train_set, y_train_set)
y_rf_prediction = cls_rf.predict(X_test_set)

print('\n confusion_matrix \n')
confusion_M_rf = confusion_matrix(y_test_set,y_rf_prediction)
print(confusion_M_rf)

print('\n Here is the classification report:')
print(classification_report(y_test_set, y_rf_prediction))

# --------------------  Random Forest -------------------------------
print('\n--------------------  Logistic Regression CLS  -------------------------------\n')

cls_logit = LogisticRegression().fit(X_train_set, y_train_set)
y_logit_prediction = cls_logit.predict(X_test_set)

print('\n confusion_matrix \n')
confusion_M_logit = confusion_matrix(y_test_set,y_logit_prediction)
print(confusion_M_logit)

print('\n Here is the classification report:')
print(classification_report(y_test_set, y_logit_prediction))

# --------------------  cluster -------------------------------
print('\n--------------------  kmeans cluster  -------------------------------\n')

true_k = 5
km = KMeans(n_clusters = true_k, init='k-means++', max_iter=100, n_init= 1)
kmini = MiniBatchKMeans(n_clusters=true_k, init='k-means++', n_init=1, init_size=1000, batch_size=1000, verbose=2)
km_model = km.fit(X_train_set)
kmini_model = kmini.fit(X_train_set)
print("For K-mean clustering ")
clustering = defaultdict(list)
for idx, label in enumerate(km_model.labels_):
    clustering[label].append(idx)
print("For K-mean Mini batch clustering ")
clustering = defaultdict(list)
for idx, label in enumerate(kmini_model.labels_):
    clustering[label].append(idx)

# --------------------  topic model -------------------------------
print('\n--------------------  topic model with gensim  -------------------------------\n')

docs = [doc for doc in sms_datas]
stoplist = stopwords.words('english')
texts = [[word for word in text.lower().split() if word not in stoplist] for text in docs]
# print(texts)

dictionary = corpora.Dictionary(texts)
# genaerate a doc-word-frequency matrix
corpus = [dictionary.doc2bow(text) for text in texts]
# print(corpus)

# generate a doc-word-tf-idf matrix
tfidf = models.TfidfModel(corpus)
corpus_tfidf = tfidf[corpus]
print('-----tf-idf value-------')
print(corpus_tfidf[0])
print(corpus_tfidf[1])
print(corpus_tfidf)

print('\n-----lsi topic model-------\n')
lsi = models.LsiModel(corpus_tfidf, id2word=dictionary, num_topics=1000)
corpus_lsi = lsi[corpus_tfidf]
print('\n-----lsi value-------\n')
print(corpus_lsi[0])
print(corpus_lsi[1])
print(lsi.print_topics(1))

print('\n-----build index for similarity-------\n')
# build index for similarity
index = similarities.MatrixSimilarity(lsi[corpus_lsi])
# query = "gold silver truck"
query = "Lol your always so convincing"
query_bow = dictionary.doc2bow(query.lower().split())
print(query_bow)
query_lsi = lsi[query_bow]
print(query_lsi)
sims = index[query_lsi]
print(list(enumerate(sims))[26])
sort_sims = sorted(enumerate(sims), key=lambda item: -item[1])[:10]
print(sort_sims)

print('\n----- LDA -------\n')
n_topics = 5
lda = models.LdaModel(corpus_tfidf, id2word = dictionary, num_topics = n_topics)
for i in range(0, n_topics):
    temp = lda.show_topic(i, 10)
    terms = []
    for term in temp:
        terms.append(str(term[0]))
    print("Top 10 terms for topic #" + str(i) + ": " + ",".join(terms))

max_stars_repo_name: Zhang-O/small

max_issues_repo_name: Zhang-O/small

max_forks_repo_name: Zhang-O/small


text:
#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""Detect keypoints with KAZE."""

import cv2
import time
import numpy as np

from airtest.utils.logger import get_logger

from .error import *  # noqa
from .utils import generate_result, check_image_valid, print_run_time
from .cal_confidence import cal_ccoeff_confidence, cal_rgb_confidence

LOGGING = get_logger(__name__)

class KeypointMatching(object):
    """基于特征点的识别基类: KAZE."""

    # 日志中的方法名
    METHOD_NAME = "KAZE"
    # 参数: FILTER_RATIO为SIFT优秀特征点过滤比例值(0-1范围，建议值0.4-0.6)
    FILTER_RATIO = 0.59
    # 参数: SIFT识别时只找出一对相似特征点时的置信度(confidence)
    ONE_POINT_CONFI = 0.5

    def __init__(self, im_search, im_source, threshold=0.8, rgb=True):
        super(KeypointMatching, self).__init__()
        self.im_source = im_source
        self.im_search = im_search
        self.threshold = threshold
        self.rgb = rgb

    def mask_kaze(self):
        """基于kaze查找多个目标区域的方法."""
        # 求出特征点后，self.im_source中获得match的那些点进行聚类
        raise NotImplementedError

    def find_all_results(self):
        """基于kaze查找多个目标区域的方法."""
        # 求出特征点后，self.im_source中获得match的那些点进行聚类
        raise NotImplementedError

    @print_run_time
    def find_best_result(self):
        """基于kaze进行图像识别，只筛选出最优区域."""
        # 第一步：检验图像是否正常：
        if not check_image_valid(self.im_source, self.im_search):
            return None

        # 第二步：获取特征点集并匹配出特征点对: 返回值 good, pypts, kp_sch, kp_src
        self.kp_sch, self.kp_src, self.good = self._get_key_points()

        # 第三步：根据匹配点对(good),提取出来识别区域:
        if len(self.good) in [0, 1]:
            # 匹配点对为0,无法提取识别区域;为1则无法获取目标区域,直接返回None作为匹配结果:
            return None
        elif len(self.good) in [2, 3]:
            # 匹配点对为2或3,根据点对求出目标区域,据此算出可信度:
            if len(self.good) == 2:
                origin_result = self._handle_two_good_points(self.kp_sch, self.kp_src, self.good)
            else:
                origin_result = self._handle_three_good_points(self.kp_sch, self.kp_src, self.good)
            # 某些特殊情况下直接返回None作为匹配结果:
            if origin_result is None:
                return origin_result
            else:
                middle_point, pypts, w_h_range = origin_result
        else:
            # 匹配点对 >= 4个，使用单矩阵映射求出目标区域，据此算出可信度：
            middle_point, pypts, w_h_range = self._many_good_pts(self.kp_sch, self.kp_src, self.good)

        # 第四步：根据识别区域，求出结果可信度，并将结果进行返回:
        # 对识别结果进行合理性校验: 小于5个像素的，或者缩放超过5倍的，一律视为不合法直接raise.
        self._target_error_check(w_h_range)
        # 将截图和识别结果缩放到大小一致,准备计算可信度
        x_min, x_max, y_min, y_max, w, h = w_h_range
        target_img = self.im_source[y_min:y_max, x_min:x_max]
        resize_img = cv2.resize(target_img, (w, h))
        confidence = self._cal_confidence(resize_img)

        best_match = generate_result(middle_point, pypts, confidence)
        LOGGING.debug("[%s] threshold=%s, result=%s" % (self.METHOD_NAME, self.threshold, best_match))
        return best_match if confidence >= self.threshold else None

    def show_match_image(self):
        """Show how the keypoints matches."""
        from random import random
        h_sch, w_sch = self.im_search.shape[:2]
        h_src, w_src = self.im_source.shape[:2]

        # first you have to do the matching
        self.find_best_result()
        # then initialize the result image:
        matching_info_img = np.zeros([max(h_sch, h_src), w_sch + w_src, 3], np.uint8)
        matching_info_img[:h_sch, :w_sch, :] = self.im_search
        matching_info_img[:h_src, w_sch:, :] = self.im_source
        # render the match image at last:
        for m in self.good:
            color = tuple([int(random() * 255) for _ in range(3)])
            cv2.line(matching_info_img, (int(self.kp_sch[m.queryIdx].pt[0]), int(self.kp_sch[m.queryIdx].pt[1])), (int(self.kp_src[m.trainIdx].pt[0] + w_sch), int(self.kp_src[m.trainIdx].pt[1])), color)

        return matching_info_img

    def _cal_confidence(self, resize_img):
        """计算confidence."""
        if self.rgb:
            confidence = cal_rgb_confidence(resize_img, self.im_search)
        else:
            confidence = cal_ccoeff_confidence(resize_img, self.im_search)
        # confidence修正
        confidence = (1 + confidence) / 2
        return confidence

    def init_detector(self):
        """Init keypoint detector object."""
        self.detector = cv2.KAZE_create()
        # create BFMatcher object:
        self.matcher = cv2.BFMatcher(cv2.NORM_L1)  # cv2.NORM_L1 cv2.NORM_L2 cv2.NORM_HAMMING(not useable)

    def get_keypoints_and_descriptors(self, image):
        """获取图像特征点和描述符."""
        keypoints, descriptors = self.detector.detectAndCompute(image, None)
        return keypoints, descriptors

    def match_keypoints(self, des_sch, des_src):
        """Match descriptors (特征值匹配)."""
        # 匹配两个图片中的特征点集，k=2表示每个特征点取出2个最匹配的对应点:
        return self.matcher.knnMatch(des_sch, des_src, k=2)

    def _get_key_points(self):
        """根据传入图像,计算图像所有的特征点,并得到匹配特征点对."""
        # 准备工作: 初始化算子
        self.init_detector()
        # 第一步：获取特征点集，并匹配出特征点对: 返回值 good, pypts, kp_sch, kp_src
        kp_sch, des_sch = self.get_keypoints_and_descriptors(self.im_search)
        kp_src, des_src = self.get_keypoints_and_descriptors(self.im_source)
        # When apply knnmatch , make sure that number of features in both test and
        #       query image is greater than or equal to number of nearest neighbors in knn match.
        if len(kp_sch) < 2 or len(kp_src) < 2:
            raise NoMatchPointError("Not enough feature points in input images !")
        # match descriptors (特征值匹配)
        matches = self.match_keypoints(des_sch, des_src)

        # good为特征点初选结果，剔除掉前两名匹配太接近的特征点，不是独特优秀的特征点直接筛除(多目标识别情况直接不适用)
        good = []
        for m, n in matches:
            if m.distance < self.FILTER_RATIO * n.distance:
                good.append(m)
        # good点需要去除重复的部分，（设定源图像不能有重复点）去重时将src图像中的重复点找出即可
        # 去重策略：允许搜索图像对源图像的特征点映射一对多，不允许多对一重复（即不能源图像上一个点对应搜索图像的多个点）
        good_diff, diff_good_point = [], [[]]
        for m in good:
            diff_point = [int(kp_src[m.trainIdx].pt[0]), int(kp_src[m.trainIdx].pt[1])]
            if diff_point not in diff_good_point:
                good_diff.append(m)
                diff_good_point.append(diff_point)
        good = good_diff

        return kp_sch, kp_src, good

    def _handle_two_good_points(self, kp_sch, kp_src, good):
        """处理两对特征点的情况."""
        pts_sch1 = int(kp_sch[good[0].queryIdx].pt[0]), int(kp_sch[good[0].queryIdx].pt[1])
        pts_sch2 = int(kp_sch[good[1].queryIdx].pt[0]), int(kp_sch[good[1].queryIdx].pt[1])
        pts_src1 = int(kp_src[good[0].trainIdx].pt[0]), int(kp_src[good[0].trainIdx].pt[1])
        pts_src2 = int(kp_src[good[1].trainIdx].pt[0]), int(kp_src[good[1].trainIdx].pt[1])

        return self._get_origin_result_with_two_points(pts_sch1, pts_sch2, pts_src1, pts_src2)

    def _handle_three_good_points(self, kp_sch, kp_src, good):
        """处理三对特征点的情况."""
        # 拿出sch和src的两个点(点1)和(点2点3的中点)，
        # 然后根据两个点原则进行后处理(注意ke_sch和kp_src以及queryIdx和trainIdx):
        pts_sch1 = int(kp_sch[good[0].queryIdx].pt[0]), int(kp_sch[good[0].queryIdx].pt[1])
        pts_sch2 = int((kp_sch[good[1].queryIdx].pt[0] + kp_sch[good[2].queryIdx].pt[0]) / 2), int(
            (kp_sch[good[1].queryIdx].pt[1] + kp_sch[good[2].queryIdx].pt[1]) / 2)
        pts_src1 = int(kp_src[good[0].trainIdx].pt[0]), int(kp_src[good[0].trainIdx].pt[1])
        pts_src2 = int((kp_src[good[1].trainIdx].pt[0] + kp_src[good[2].trainIdx].pt[0]) / 2), int(
            (kp_src[good[1].trainIdx].pt[1] + kp_src[good[2].trainIdx].pt[1]) / 2)
        return self._get_origin_result_with_two_points(pts_sch1, pts_sch2, pts_src1, pts_src2)

    def _many_good_pts(self, kp_sch, kp_src, good):
        """特征点匹配点对数目>=4个，可使用单矩阵映射,求出识别的目标区域."""
        sch_pts, img_pts = np.float32([kp_sch[m.queryIdx].pt for m in good]).reshape(
            -1, 1, 2), np.float32([kp_src[m.trainIdx].pt for m in good]).reshape(-1, 1, 2)
        # M是转化矩阵
        M, mask = self._find_homography(sch_pts, img_pts)
        matches_mask = mask.ravel().tolist()
        # 从good中间筛选出更精确的点(假设good中大部分点为正确的，由ratio=0.7保障)
        selected = [v for k, v in enumerate(good) if matches_mask[k]]

        # 针对所有的selected点再次计算出更精确的转化矩阵M来
        sch_pts, img_pts = np.float32([kp_sch[m.queryIdx].pt for m in selected]).reshape(
            -1, 1, 2), np.float32([kp_src[m.trainIdx].pt for m in selected]).reshape(-1, 1, 2)
        M, mask = self._find_homography(sch_pts, img_pts)
        # 计算四个角矩阵变换后的坐标，也就是在大图中的目标区域的顶点坐标:
        h, w = self.im_search.shape[:2]
        h_s, w_s = self.im_source.shape[:2]
        pts = np.float32([[0, 0], [0, h - 1], [w - 1, h - 1], [w - 1, 0]]).reshape(-1, 1, 2)
        dst = cv2.perspectiveTransform(pts, M)

        # trans numpy arrary to python list: [(a, b), (a1, b1), ...]
        def cal_rect_pts(dst):
            return [tuple(npt[0]) for npt in dst.astype(int).tolist()]

        pypts = cal_rect_pts(dst)
        # 注意：虽然4个角点有可能越出source图边界，但是(根据精确化映射单映射矩阵M线性机制)中点不会越出边界
        lt, br = pypts[0], pypts[2]
        middle_point = int((lt[0] + br[0]) / 2), int((lt[1] + br[1]) / 2)
        # 考虑到算出的目标矩阵有可能是翻转的情况，必须进行一次处理，确保映射后的“左上角”在图片中也是左上角点：
        x_min, x_max = min(lt[0], br[0]), max(lt[0], br[0])
        y_min, y_max = min(lt[1], br[1]), max(lt[1], br[1])
        # 挑选出目标矩形区域可能会有越界情况，越界时直接将其置为边界：
        # 超出左边界取0，超出右边界取w_s-1，超出下边界取0，超出上边界取h_s-1
        # 当x_min小于0时，取0。  x_max小于0时，取0。
        x_min, x_max = int(max(x_min, 0)), int(max(x_max, 0))
        # 当x_min大于w_s时，取值w_s-1。  x_max大于w_s-1时，取w_s-1。
        x_min, x_max = int(min(x_min, w_s - 1)), int(min(x_max, w_s - 1))
        # 当y_min小于0时，取0。  y_max小于0时，取0。
        y_min, y_max = int(max(y_min, 0)), int(max(y_max, 0))
        # 当y_min大于h_s时，取值h_s-1。  y_max大于h_s-1时，取h_s-1。
        y_min, y_max = int(min(y_min, h_s - 1)), int(min(y_max, h_s - 1))
        # 目标区域的角点，按左上、左下、右下、右上点序：(x_min,y_min)(x_min,y_max)(x_max,y_max)(x_max,y_min)
        pts = np.float32([[x_min, y_min], [x_min, y_max], [
                         x_max, y_max], [x_max, y_min]]).reshape(-1, 1, 2)
        pypts = cal_rect_pts(pts)

        return middle_point, pypts, [x_min, x_max, y_min, y_max, w, h]

    def _get_origin_result_with_two_points(self, pts_sch1, pts_sch2, pts_src1, pts_src2):
        """返回两对有效匹配特征点情形下的识别结果."""
        # 先算出中心点(在self.im_source中的坐标)：
        middle_point = [int((pts_src1[0] + pts_src2[0]) / 2), int((pts_src1[1] + pts_src2[1]) / 2)]
        pypts = []
        # 如果特征点同x轴或同y轴(无论src还是sch中)，均不能计算出目标矩形区域来，此时返回值同good=1情形
        if pts_sch1[0] == pts_sch2[0] or pts_sch1[1] == pts_sch2[1] or pts_src1[0] == pts_src2[0] or pts_src1[1] == pts_src2[1]:
            return None
        # 计算x,y轴的缩放比例：x_scale、y_scale，从middle点扩张出目标区域:(注意整数计算要转成浮点数结果!)
        h, w = self.im_search.shape[:2]
        h_s, w_s = self.im_source.shape[:2]
        x_scale = abs(1.0 * (pts_src2[0] - pts_src1[0]) / (pts_sch2[0] - pts_sch1[0]))
        y_scale = abs(1.0 * (pts_src2[1] - pts_src1[1]) / (pts_sch2[1] - pts_sch1[1]))
        # 得到scale后需要对middle_point进行校正，并非特征点中点，而是映射矩阵的中点。
        sch_middle_point = int((pts_sch1[0] + pts_sch2[0]) / 2), int((pts_sch1[1] + pts_sch2[1]) / 2)
        middle_point[0] = middle_point[0] - int((sch_middle_point[0] - w / 2) * x_scale)
        middle_point[1] = middle_point[1] - int((sch_middle_point[1] - h / 2) * y_scale)
        middle_point[0] = max(middle_point[0], 0)  # 超出左边界取0  (图像左上角坐标为0,0)
        middle_point[0] = min(middle_point[0], w_s - 1)  # 超出右边界取w_s-1
        middle_point[1] = max(middle_point[1], 0)  # 超出上边界取0
        middle_point[1] = min(middle_point[1], h_s - 1)  # 超出下边界取h_s-1

        # 计算出来rectangle角点的顺序：左上角->左下角->右下角->右上角， 注意：暂不考虑图片转动
        # 超出左边界取0, 超出右边界取w_s-1, 超出下边界取0, 超出上边界取h_s-1
        x_min, x_max = int(max(middle_point[0] - (w * x_scale) / 2, 0)), int(
            min(middle_point[0] + (w * x_scale) / 2, w_s - 1))
        y_min, y_max = int(max(middle_point[1] - (h * y_scale) / 2, 0)), int(
            min(middle_point[1] + (h * y_scale) / 2, h_s - 1))
        # 目标矩形的角点按左上、左下、右下、右上的点序：(x_min,y_min)(x_min,y_max)(x_max,y_max)(x_max,y_min)
        pts = np.float32([[x_min, y_min], [x_min, y_max], [x_max, y_max], [x_max, y_min]]).reshape(-1, 1, 2)
        for npt in pts.astype(int).tolist():
            pypts.append(tuple(npt[0]))

        return middle_point, pypts, [x_min, x_max, y_min, y_max, w, h]

    def _find_homography(self, sch_pts, src_pts):
        """多组特征点对时，求取单向性矩阵."""
        try:
            M, mask = cv2.findHomography(sch_pts, src_pts, cv2.RANSAC, 5.0)
        except Exception:
            import traceback
            traceback.print_exc()
            raise HomographyError("OpenCV error in _find_homography()...")
        else:
            if mask is None:
                raise HomographyError("In _find_homography(), find no transfomation matrix...")
            else:
                return M, mask

    def _target_error_check(self, w_h_range):
        """校验识别结果区域是否符合常理."""
        x_min, x_max, y_min, y_max, w, h = w_h_range
        tar_width, tar_height = x_max - x_min, y_max - y_min
        # 如果src_img中的矩形识别区域的宽和高的像素数＜5，则判定识别失效。认为提取区域待不可能小于5个像素。(截图一般不可能小于5像素)
        if tar_width < 5 or tar_height < 5:
            raise MatchResultCheckError("In src_image, Taget area: width or height < 5 pixel.")
        # 如果矩形识别区域的宽和高，与sch_img的宽高差距超过5倍(屏幕像素差不可能有5倍)，认定为识别错误。
        if tar_width < 0.2 * w or tar_width > 5 * w or tar_height < 0.2 * h or tar_height > 5 * h:
            raise MatchResultCheckError("Target area is 5 times bigger or 0.2 times smaller than sch_img.")

ext: py

max_stars_repo_name: Aracoix/Airtest

max_issues_repo_name: Aracoix/Airtest

max_forks_repo_name: Aracoix/Airtest

reason: import numpy


text:
# imposta la regione scelta ed esegui lo script

interested_region = "Veneto"

# -----------------------------------------------------------------------------

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn

seaborn.set_style("dark")

# auxiliary functions

def parse_data(s):
    s = s.split(" ")[0]
    mese = s.split("-")[1]
    giorno = s.split("-")[2]

    return giorno + "/" + str(int(mese))

# 0: regione_nuovi/tamponi
# 1: provincia totali e logaritmo totali TODO
plot_type = 0

regioni_csv = pd.read_csv("dati-regioni/dpc-covid19-ita-regioni.csv")

if plot_type == 0:
    # --------------- OBTAIN DATA -------------------------------------

    regione_csv = regioni_csv[regioni_csv["denominazione_regione"]==interested_region]
    # ottengo gioni
    giorni = list(map(parse_data, regione_csv["data"].values))

    # nuovi casi all'interno della regione
    totale_casi = regione_csv["totale_casi"].values
    nuovi_casi_positivi = [totale_casi[0]]
    for g in range(len(totale_casi) - 1):
        nuovi_casi_positivi += [totale_casi[g+1] - totale_casi[g]]

    # ottengo il numero dei nuovi tamponi giornalieri
    numero_tamponi = regione_csv["tamponi"].values
    delta_tamponi = [numero_tamponi[0]]
    for g in range(len(numero_tamponi) - 1):
        delta_tamponi += [numero_tamponi[g+1] - numero_tamponi[g]]

    # percentuale nuovi_positivi/nuovi_tamponi
    div = lambda x: x[0]/x[1] if 0<(x[0]/x[1])<1 else None
    ratio_nuovi_pos_nuovi_tamp = list(map(div, list(zip(nuovi_casi_positivi, delta_tamponi))))

    # --------------- END OBTAIN DATA ---------------------------
    w = 0.3
    fig, ax1 = plt.subplots()

    ind = np.arange(len(giorni))

    ax1.set_xlabel("Giorni")
    ax1.set_ylabel("Unità")
    bar_pos = ax1.bar(ind+w, nuovi_casi_positivi, color="tab:red", width=w)
    bar_tamp = ax1.bar(ind, delta_tamponi, color="tab:grey", width=w)
    ax1.tick_params(axis='y')

    ax1.set_xticks(ind+(w/2))
    ax1.set_xticklabels(giorni)
    ax1.legend( (bar_pos, bar_tamp), ('Nuovi casi positivi', 'Nuovi tamponi') )
    ax1.set_xticks([])

    ax2 = ax1.twinx()  # instantiate a second axes that shares the same x-axis

    color = 'tab:blue'
    ax2.set_ylabel('Nuovi casi positivi / nuovi tamponi (al giorno)', color=color)  # we already handled the x-label with ax1
    line_perc = ax2.plot(giorni, ratio_nuovi_pos_nuovi_tamp, color=color)
    ax2.tick_params(axis='y', labelcolor=color)

    fig.tight_layout()  # otherwise the right y-label is slightly clipped
    fig.set_size_inches(20, 5)
    plt.title(interested_region + ": confronto giornaliero tra nuovi casi positivi e nuovi tamponi")
    plt.show()


text:
# -*- coding: utf-8 -*-
import numpy as np
import torch

from scilpy.tracking.seed import SeedGenerator

class DWIMLSeedGenerator(SeedGenerator):
    """
    Seed generator with added methods to generate many seeds instead of one
    at the time, for GPU processing of many streamlines at once.
    """

    def __init__(self, data, voxres, device=None):
        super().__init__(data, voxres)

        self.data = torch.tensor(self.data)
        self.device = device
        if device is not None:
            self.move_to(device)

    def move_to(self, device):
        self.data = self.data.to(device=device)
        self.device = device

    def get_next_n_pos(self, random_generator, indices, which_seeds):
        """
        Generate the next n seed positions (Space=voxmm, origin=corner).
        Heavy, should be used on GPU.

        Parameters
        ----------
        random_generator : numpy random generator
            Initialized numpy number generator.
        indices : List
            Indices of current seeding map.
        which_seeds : List[int]
            Seed numbers (i.e. IDs) to be processed.

        Return
        ------
        seed_pos: List[tuple]
            Positions of next seeds expressed in mm.
        """
        len_seeds = len(self.seeds)
        if len_seeds == 0:
            return []

        voxel_dim = np.asarray(self.voxres)

        # Voxel selection from the seeding mask
        inds = torch.fmod(which_seeds, len_seeds)
        x, y, z = self.seeds[indices[inds]]

        # Sub-voxel initial positioning
        n = len(which_seeds)
        r_x = random_generator.uniform(0, voxel_dim[0], size=n)
        r_y = random_generator.uniform(0, voxel_dim[1], size=n)
        r_z = random_generator.uniform(0, voxel_dim[2], size=n)

        return x * self.voxres[0] + r_x, y * self.voxres[1] \
            + r_y, z * self.voxres[2] + r_z


text:
import numpy as np
import os

cbox = np.array([[0, 70.4], [-40, 40], [-3, 2]])

def inverse_rigid_trans(Tr):
    """ Inverse a rigid body transform matrix (3x4 as [R|t])
        [R'|-R't; 0|1]
    """
    inv_Tr = np.zeros_like(Tr)  # 3x4
    inv_Tr[0:3, 0:3] = np.transpose(Tr[0:3, 0:3])
    inv_Tr[0:3, 3] = np.dot(-np.transpose(Tr[0:3, 0:3]), Tr[0:3, 3])
    return inv_Tr

def get_depth_pt3d(depth):
    pt3d = []
    for i in range(depth.shape[0]):
        for j in range(depth.shape[1]):
            pt3d.append([i, j, depth[i, j]])
    return np.array(pt3d)

class Calibration(object):
    """ Calibration matrices and utils
        3d XYZ in <label>.txt are in rect camera coord.
        2d box xy are in image2 coord
        Points in <lidar>.bin are in Velodyne coord.

        y_image2 = P^2_rect * x_rect
        y_image2 = P^2_rect * R0_rect * Tr_velo_to_cam * x_velo
        x_ref = Tr_velo_to_cam * x_velo
        x_rect = R0_rect * x_ref

        P^2_rect = [f^2_u,  0,      c^2_u,  -f^2_u b^2_x;
                    0,      f^2_v,  c^2_v,  -f^2_v b^2_y;
                    0,      0,      1,      0]
                 = K * [1|t]

        image2 coord:
         ----> x-axis (u)
        |
        |
        v y-axis (v)

        velodyne coord:
        front x, left y, up z

        rect/ref camera coord:
        right x, down y, front z

        Ref (KITTI paper): http://www.cvlibs.net/publications/Geiger2013IJRR.pdf

        TODO(rqi): do matrix multiplication only once for each projection.
    """

    def __init__(self, calib_filepath, from_video=False):
        if from_video:
            calibs = self.read_calib_from_video(calib_filepath)
        else:
            calibs = self.read_calib_file(calib_filepath)
        # Projection matrix from rect camera coord to image2 coord
        self.P = calibs["P2"]
        self.P = np.reshape(self.P, [3, 4])
        # Rigid transform from Velodyne coord to reference camera coord
        self.V2C = calibs["Tr_velo_to_cam"]
        self.V2C = np.reshape(self.V2C, [3, 4])
        self.C2V = inverse_rigid_trans(self.V2C)
        # Rotation from reference camera coord to rect camera coord
        self.R0 = calibs["R0_rect"]
        self.R0 = np.reshape(self.R0, [3, 3])

        # Camera intrinsics and extrinsics
        self.c_u = self.P[0, 2]
        self.c_v = self.P[1, 2]
        self.f_u = self.P[0, 0]
        self.f_v = self.P[1, 1]
        self.b_x = self.P[0, 3] / (-self.f_u)  # relative
        self.b_y = self.P[1, 3] / (-self.f_v)

    def read_calib_file(self, filepath):
        """ Read in a calibration file and parse into a dictionary.
        Ref: https://github.com/utiasSTARS/pykitti/blob/master/pykitti/utils.py
        """
        data = {}
        with open(filepath, "r") as f:
            for line in f.readlines():
                line = line.rstrip()
                if len(line) == 0:
                    continue
                key, value = line.split(":", 1)
                # The only non-float values in these files are dates, which
                # we don't care about anyway
                try:
                    data[key] = np.array([float(x) for x in value.split()])
                except ValueError:
                    pass

        return data

    def read_calib_from_video(self, calib_root_dir):
        """ Read calibration for camera 2 from video calib files.
            there are calib_cam_to_cam and calib_velo_to_cam under the calib_root_dir
        """
        data = {}
        cam2cam = self.read_calib_file(
            os.path.join(calib_root_dir, "calib_cam_to_cam.txt")
        )
        velo2cam = self.read_calib_file(
            os.path.join(calib_root_dir, "calib_velo_to_cam.txt")
        )
        Tr_velo_to_cam = np.zeros((3, 4))
        Tr_velo_to_cam[0:3, 0:3] = np.reshape(velo2cam["R"], [3, 3])
        Tr_velo_to_cam[:, 3] = velo2cam["T"]
        data["Tr_velo_to_cam"] = np.reshape(Tr_velo_to_cam, [12])
        data["R0_rect"] = cam2cam["R_rect_00"]
        data["P2"] = cam2cam["P_rect_02"]
        return data

    def cart2hom(self, pts_3d):
        """ Input: nx3 points in Cartesian
            Oupput: nx4 points in Homogeneous by pending 1
        """
        n = pts_3d.shape[0]
        pts_3d_hom = np.hstack((pts_3d, np.ones((n, 1))))
        return pts_3d_hom

    # ===========================
    # ------- 3d to 3d ----------
    # ===========================
    def project_velo_to_ref(self, pts_3d_velo):
        pts_3d_velo = self.cart2hom(pts_3d_velo)  # nx4
        return np.dot(pts_3d_velo, np.transpose(self.V2C))

    def project_ref_to_velo(self, pts_3d_ref):
        pts_3d_ref = self.cart2hom(pts_3d_ref)  # nx4
        return np.dot(pts_3d_ref, np.transpose(self.C2V))

    def project_rect_to_ref(self, pts_3d_rect):
        """ Input and Output are nx3 points """
        return np.transpose(np.dot(np.linalg.inv(self.R0), np.transpose(pts_3d_rect)))

    def project_ref_to_rect(self, pts_3d_ref):
        """ Input and Output are nx3 points """
        return np.transpose(np.dot(self.R0, np.transpose(pts_3d_ref)))

    def project_rect_to_velo(self, pts_3d_rect):
        """ Input: nx3 points in rect camera coord.
            Output: nx3 points in velodyne coord.
        """
        pts_3d_ref = self.project_rect_to_ref(pts_3d_rect)
        return self.project_ref_to_velo(pts_3d_ref)

    def project_velo_to_rect(self, pts_3d_velo):
        pts_3d_ref = self.project_velo_to_ref(pts_3d_velo)
        return self.project_ref_to_rect(pts_3d_ref)

    # ===========================
    # ------- 3d to 2d ----------
    # ===========================
    def project_rect_to_image(self, pts_3d_rect):
        """ Input: nx3 points in rect camera coord.
            Output: nx2 points in image2 coord.
        """
        pts_3d_rect = self.cart2hom(pts_3d_rect)
        pts_2d = np.dot(pts_3d_rect, np.transpose(self.P))  # nx3
        pts_2d[:, 0] /= pts_2d[:, 2]
        pts_2d[:, 1] /= pts_2d[:, 2]
        return pts_2d[:, 0:2]

    def project_velo_to_image(self, pts_3d_velo):
        """ Input: nx3 points in velodyne coord.
            Output: nx2 points in image2 coord.
        """
        pts_3d_rect = self.project_velo_to_rect(pts_3d_velo)
        return self.project_rect_to_image(pts_3d_rect)

    def project_8p_to_4p(self, pts_2d):
        x0 = np.min(pts_2d[:, 0])
        x1 = np.max(pts_2d[:, 0])
        y0 = np.min(pts_2d[:, 1])
        y1 = np.max(pts_2d[:, 1])
        x0 = max(0, x0)
        # x1 = min(x1, proj.image_width)
        y0 = max(0, y0)
        # y1 = min(y1, proj.image_height)
        return np.array([x0, y0, x1, y1])

    def project_velo_to_4p(self, pts_3d_velo):
        """ Input: nx3 points in velodyne coord.
            Output: 4 points in image2 coord.
        """
        pts_2d_velo = self.project_velo_to_image(pts_3d_velo)
        return self.project_8p_to_4p(pts_2d_velo)

    # ===========================
    # ------- 2d to 3d ----------
    # ===========================
    def project_image_to_rect(self, uv_depth):
        """ Input: nx3 first two channels are uv, 3rd channel
                   is depth in rect camera coord.
            Output: nx3 points in rect camera coord.
        """
        n = uv_depth.shape[0]
        x = ((uv_depth[:, 0] - self.c_u) * uv_depth[:, 2]) / self.f_u + self.b_x
        y = ((uv_depth[:, 1] - self.c_v) * uv_depth[:, 2]) / self.f_v + self.b_y
        pts_3d_rect = np.zeros((n, 3))
        pts_3d_rect[:, 0] = x
        pts_3d_rect[:, 1] = y
        pts_3d_rect[:, 2] = uv_depth[:, 2]
        return pts_3d_rect

    def project_image_to_velo(self, uv_depth):
        pts_3d_rect = self.project_image_to_rect(uv_depth)
        return self.project_rect_to_velo(pts_3d_rect)

    def project_depth_to_velo(self, depth, constraint_box=True):
        depth_pt3d = get_depth_pt3d(depth)
        depth_UVDepth = np.zeros_like(depth_pt3d)
        depth_UVDepth[:, 0] = depth_pt3d[:, 1]
        depth_UVDepth[:, 1] = depth_pt3d[:, 0]
        depth_UVDepth[:, 2] = depth_pt3d[:, 2]
        # print("depth_pt3d:",depth_UVDepth.shape)
        depth_pc_velo = self.project_image_to_velo(depth_UVDepth)
        # print("dep_pc_velo:",depth_pc_velo.shape)
        if constraint_box:
            depth_box_fov_inds = (
                (depth_pc_velo[:, 0] < cbox[0][1])
                & (depth_pc_velo[:, 0] >= cbox[0][0])
                & (depth_pc_velo[:, 1] < cbox[1][1])
                & (depth_pc_velo[:, 1] >= cbox[1][0])
                & (depth_pc_velo[:, 2] < cbox[2][1])
                & (depth_pc_velo[:, 2] >= cbox[2][0])
            )
            depth_pc_velo = depth_pc_velo[depth_box_fov_inds]
        return depth_pc_velo

max_stars_repo_name: qixuxiang/EasyPointPillars

max_issues_repo_name: chisyliu/EasyPointPillars

max_forks_repo_name: chisyliu/EasyPointPillars


text:
'''
Copyright 2022 Airbus SAS

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
'''
from sos_trades_core.study_manager.study_manager import StudyManager
from numpy import array
import pandas as pd

class Study(StudyManager):

        def __init__(self, execution_engine=None):
            super().__init__(__file__, execution_engine=execution_engine)

        def setup_usecase(self):
            ns = f'{self.study_name}'
            sc_name = "SellarOptimScenario.SellarCoupling"
            sc_name_ncoupling = "SellarOptimScenario"
            dspace_dict = {'variable': ['x', 'z', 'y_1', 'y_2'],
                           'value': [[1.], [5., 2.], [5.], [1.]],
                           'lower_bnd': [[0.], [-10., 0.], [-100.], [-100.]],
                           'upper_bnd': [[10.], [10., 10.], [100.], [100.]],
                           'enable_variable': [True, True, True, True],
                           'activated_elem': [[True], [True, True], [True], [True]]}
            #                   'type' : ['float',['float','float'],'float','float']
            dspace = pd.DataFrame(dspace_dict)

            disc_dict = {}
            # Optim inputs
            disc_dict[f'{ns}.SellarOptimScenario.max_iter'] = 100
            # SLSQP, NLOPT_SLSQP
            disc_dict[f'{ns}.SellarOptimScenario.algo'] = "SLSQP"
            disc_dict[f'{ns}.SellarOptimScenario.design_space'] = dspace
            # TODO: what's wrong with IDF
            disc_dict[f'{ns}.SellarOptimScenario.formulation'] = 'MDF'
            # f'{ns}.SellarOptimScenario.obj'
            disc_dict[f'{ns}.SellarOptimScenario.objective_name'] = 'obj'
            disc_dict[f'{ns}.SellarOptimScenario.ineq_constraints'] = [
                'c_1', 'c_2']
            # f'{ns}.SellarOptimScenario.c_1', f'{ns}.SellarOptimScenario.c_2']

            disc_dict[f'{ns}.SellarOptimScenario.algo_options'] = {"ftol_rel": 1e-10,
                                                                   "ineq_tolerance": 2e-3,
                                                                   "normalize_design_space": False}

            # Sellar inputs
            disc_dict[f'{ns}.{sc_name}.x'] = 1.
            #         disc_dict[f'{ns}.{sc_name}.y_1'] = array([1.])
            #         disc_dict[f'{ns}.{sc_name}.y_2'] = array([1.])
            disc_dict[f'{ns}.{sc_name}.z'] = array([1., 1.])
            disc_dict[f'{ns}.{sc_name}.Sellar_Problem.local_dv'] = 10.
            scatter_scenario_name = 'optimization scenarios'
            scenario_list = ['a=0-1', 'a=0-2']
            disc_dict[f'{ns}.{scatter_scenario_name}.scenario_list'] = scenario_list

            for scen in scenario_list:
                disc_dict[f'{ns}.{scatter_scenario_name}.{scen}.{sc_name}.Sellar_Problem.local_dv'] = 5.
                disc_dict[f'{ns}.{scatter_scenario_name}.{scen}.{sc_name}.z'] = array([2., 1.])
                disc_dict[f'{ns}.{scatter_scenario_name}.{scen}.{sc_name_ncoupling}.max_iter'] = 200
                disc_dict[f'{ns}.{scatter_scenario_name}.{scen}.{sc_name}.x'] = 500.

                # SLSQP, NLOPT_SLSQP
                disc_dict[f'{ns}.{scatter_scenario_name}.{scen}.{sc_name_ncoupling}.algo'] = "SLSQP"
                disc_dict[f'{ns}.{scatter_scenario_name}.{scen}.{sc_name_ncoupling}.design_space'] = dspace
                # TODO: what's wrong with IDF
                disc_dict[f'{ns}.{scatter_scenario_name}.{scen}.{sc_name_ncoupling}.formulation'] = 'MDF'
                # f'{ns}.SellarOptimScenario.obj'
                disc_dict[f'{ns}.{scatter_scenario_name}.{scen}.{sc_name_ncoupling}.objective_name'] = 'obj'
                disc_dict[f'{ns}.{scatter_scenario_name}.{scen}.{sc_name_ncoupling}.ineq_constraints'] = [
                    'c_1', 'c_2']
                # f'{ns}.SellarOptimScenario.c_1', f'{ns}.SellarOptimScenario.c_2']

                disc_dict[f'{ns}.{scatter_scenario_name}.{scen}.{sc_name_ncoupling}.algo_options'] = {"ftol_rel": 1e-10,
                                                                       "ineq_tolerance": 2e-3,
                                                                       "normalize_design_space": False}

            return [disc_dict]

if '__main__' == __name__:
    uc_cls = Study()
    uc_cls.load_data()
    uc_cls.execution_engine.display_treeview_nodes(display_variables=False)
    uc_cls.run()

max_stars_repo_name: os-climate/sostrades-core

max_issues_repo_name: os-climate/sostrades-core

max_forks_repo_name: os-climate/sostrades-core

reason: from numpy


text:
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Mon Apr 27 15:45:13 2020

A scratchpad for solving for beamline propagation

@author: twguest
"""

###############################################################################
import sys
sys.path.append("/opt/WPG/") # LOCAL PATH
sys.path.append("/gpfs/exfel/data/user/guestt/WPG") # DESY MAXWELL PATH

sys.path.append("/opt/spb_model") # LOCAL PATH
sys.path.append("/gpfs/exfel/data/user/guestt/spb_model") # DESY MAXWELL PATH
###############################################################################
###############################################################################

import time

import numpy as np
import matplotlib as mpl
from copy import copy

from felpy.model.beamline import Beamline
from wpg.srwlib import SRWLOptD

from model.src.coherent import construct_SA1_wavefront
from model.beamline.structure import Instrument, propagation_parameters
from model.materials.load_refl import load_refl, get_refl

from wpg.wpg_uti_wf import plot_intensity_map as plotIntensity
from wpg.wpg_uti_wf import plot_intensity_qmap as plotPhase
from wpg.wpg_uti_wf import calculate_fwhm

from matplotlib import pyplot as plt

def getFocusSize(ekev, q, focus = 'micron'):

    wfr = construct_SA1_wavefront(1024, 1024, ekev, q)

    spb = Instrument(overwrite_mirrors =  True)

    spb.setupHOMs(ekev, 2.2e-03)
    spb.setupKBs(ekev, 3.5e-03)

    spb.build_elements(focus = focus)
<<<<<<< HEAD
    spb.build_beamline(focus = focus)
=======
    spb.buildBeamline(focus = focus)
>>>>>>> 108cfb9b6fc97d3841ee1db54862523eee5b184e
    bl = spb.get_beamline()

    bl.propagate(wfr)

    meas = calculate_fwhm(wfr)

    fwhm_x = meas['fwhm_x']
    fwhm_y = meas['fwhm_y']

    return fwhm_x, fwhm_y

def plot(energyrange, data, estr, qstr, focus, outdir = "../../tmp/"):

    fig = plt.figure()
    ax = fig.add_subplot(111)

    if focus == "micron":
        focstr = "Micron-KB "
    elif focus == "nano":
        focstr = "Nano-KB "

    ax.set_title(focstr + "Focal Spot Size: q = {} nC".format(qstr))
    ax.set_xlabel("Energy (keV)")
    ax.set_ylabel("Focal Spot Width ($\mu$m)")

    ax.scatter(energyrange, data[0], c = 'r')
    ax.scatter(energyrange, data[1], c = 'b')

    ax.set_ylim()

    plt.legend(['Horizontal Beam Width', 'Vertical Beam Width'])

    fig.savefig(outdir + "FocalSpotSize_{}_{}nC.png".format(focus, qstr))

if __name__ == '__main__':

    focus = 'micron'

    energyrange = np.linspace(6.0, 15.0, 9)
    qrange = [0.1, 0.25, 0.50]

    for q in qrange:
        qstr = str(q)
        _X, _Y = [], []
        for ekev in energyrange:

            estr = str(ekev).replace(".","-")

            fwhm_x, fwhm_y = getFocusSize(ekev, q, focus = focus)

            _X.append(fwhm_x*1e6)
            _Y.append(fwhm_y*1e6)

        plot(energyrange, [_X, _Y], estr, qstr, focus, outdir = "waist_size/")

max_stars_repo_name: twguest/FELPy

max_issues_repo_name: twguest/FELPy

max_forks_repo_name: twguest/FELPy


text:
from .utils import SuppressStderr

import pyaudio
import numpy as np

# RAII wrapper for PyAudio
# This properly terminates the PyAudio object when it goes out of scope, and it
# suppresses all the annoying stderr output when the object is created
class Audio(pyaudio.PyAudio):
    def __init__(self, *args, **kwargs):
        with SuppressStderr():
            super().__init__(*args, **kwargs)

    def __del__(self):
        self.terminate()

    def get_stream(self, input_device, output_device, frame_rate, buffer_size, callback):
        return self.open(
            input=True,
            input_device_index=input_device,
            output=True,
            output_device_index=output_device,
            rate=frame_rate,
            frames_per_buffer=buffer_size,
            stream_callback=callback,
            format=pyaudio.paFloat32,
            channels=1)

# Holds IDs and names of audio devices
class DevicesInfo():
    def __init__(self):
        self.input = {}
        self.output = {}

    def __repr__(self):
        msg = 'DevicesInfo(\n'
        msg += '  input:\n'
        for device_id, device_name in self.input.items():
            msg += f'    {device_id}: "{device_name}"\n'
        msg += '  output:\n'
        for device_id, device_name in self.output.items():
            msg += f'    {device_id}: "{device_name}"\n'
        msg += ')'
        return msg

    def __str__(self):
        return repr(self)

# Get IDs and names of available audio devices
def get_devices():
    devices = DevicesInfo()
    p = Audio()
    info = p.get_host_api_info_by_index(0)
    num_devices = info.get('deviceCount')

    for device_id in range(num_devices):
        device_info = p.get_device_info_by_host_api_device_index(0, device_id)
        device_name = device_info.get('name')

        if device_info.get('maxInputChannels') > 0:
            devices.input[device_id] = device_name

        if device_info.get('maxOutputChannels') > 0:
            devices.output[device_id] = device_name

    return devices

def play_signal(signal, output_device, frame_rate):
    p = Audio()
    stream = p.open(
        output=True,
        output_device_index=output_device,
        rate=frame_rate,
        format=pyaudio.paFloat32,
        channels=1)
    stream.write(signal.tobytes())

    stream.stop_stream()
    stream.close()

def run(input_signal, frame_rate, input_device, output_device):
    p = Audio()
    buffer_size = 1024

    # Signal buffers read from the output of the DUT
    output_buffers = []

    cur_frame = 0

    # This function is called for each audio stream buffer
    def callback(output_buffer, frame_count, time_info, status_flags):
        nonlocal cur_frame
        output_buffers.append(output_buffer)

        if cur_frame >= input_signal.size:
            flag = pyaudio.paComplete
            input_buffer = np.zeros(frame_count).astype(np.float32).tobytes()
        else:
            flag = pyaudio.paContinue
            input_buffer = input_signal[cur_frame : cur_frame + frame_count].tobytes()

        cur_frame += frame_count

        return (input_buffer, flag)

    # TODO: Make an RAII wrapper for stream
    stream = p.get_stream(
        input_device,
        output_device,
        frame_rate,
        buffer_size,
        callback)

    while stream.is_active():
        pass

    stream.stop_stream()
    stream.close()

    output_signal = np.concatenate(
        [np.frombuffer(buffer, dtype=np.float32) for buffer in output_buffers])

    return output_signal

max_stars_repo_name: kurtamohler/freqbench

max_issues_repo_name: kurtamohler/freqbench

max_forks_repo_name: kurtamohler/freqbench


text:
"""Benchmark creating CellGrids"""
import timeit
import numpy as np

import cellgrid

stp = """\
import cellgrid
import numpy as np

coords = np.random.random({n} * 3).reshape({n}, 3) * 10
box = np.ones(3) * 10
"""

for n in [100, 1000, 10000, 100000]:
    val = timeit.timeit('cg = cellgrid.CellGrid(box, 2, coords)',
                        setup=stp.format(n=n), number=100)
    print n, val

max_stars_repo_name: ayushsuhane/cellgrid

max_issues_repo_name: ayushsuhane/cellgrid

max_forks_repo_name: ayushsuhane/cellgrid


text:
# -*- coding: utf-8 -*-
import numpy as np
import matplotlib.pyplot as plt

def gd(X, Y, alpha = 0.01, epsilon=1e-32):
    m = len(X)
    a, b = 0, 0
    sse0 = 0
    while True:
        grad_a, grad_b = 0, 0
        for i in range(m):
            diff = a * X[i] + b - Y[i]
            grad_a += X[i] * diff
            grad_b += diff

        grad_a = grad_a / m
        grad_b = grad_b / m

        a -= alpha * grad_a
        b -= alpha * grad_b

        sse1 = 0
        for j in range(m):
            sse1 += (a * X[j] + b - Y[j]) ** 2 / (2 * m)

        if abs(sse0 - sse1) < epsilon:
            break
        else:
            sse0 = sse1
    return a, b

X = np.array([1, 2, 3, 4, 5, 6, 7, 8, 9])
Y = np.array([0.199, 0.389, 0.580, 0.783, 0.980, 1.177, 1.380, 1.575, 1.771])

a1, b1 = gd(X, Y, 0.05, 1e-6)
a2, b2 = gd(X, Y, 0.05, 1e-8)
a3, b3 = gd(X, Y, 0.05, 1e-32)

print('y = {0} * x + {1}'.format(a1, b1))
print('y = {0} * x + {1}'.format(a2, b2))
print('y = {0} * x + {1}'.format(a3, b3))

A = np.vstack([X, np.ones(len(X))]).T
a, b = np.linalg.lstsq(A, Y)[0]

print('y = {0} * x + {1}'.format(a, b))

x = np.array(X)
plt.plot(x, Y, 'o', label='Original data', markersize=5)
plt.plot(x, a1 * x + b1, 'r--', label='Fitted line')
plt.plot(x, a2 * x + b2, 'g--', label='Fitted line')
plt.plot(x, a3 * x + b3, 'b--', label='Fitted line')
plt.plot(x, a * x + b, 'm--', label='Fitted line')
plt.show()

max_stars_repo_name: jameszhan/notes-ml

max_issues_repo_name: jameszhan/notes-ml

max_forks_repo_name: jameszhan/notes-ml


text:
import numpy as np
import os
from utils import create_dir
import tqdm

path_to_pointnet_data = "/mnt/raid/pablo/data/FlyingThings3D/pointcloud_voxelgrid_15_15_10_10_5_25/pointnet_data"
path_to_pointnet_data_features = "/mnt/raid/pablo/data/FlyingThings3D/pointcloud_voxelgrid_15_15_10_10_5_25/pointnet_data_features"

data_splits = ["TEST", "TRAIN"]
letters = ["A", "B", "C"]

SPLIT = "TRAIN"
LETTER = "A"
NUMBER = "0000"
just_one_sample = True

for data_split in data_splits:
    # if data_split != SPLIT and just_one_sample:
    #     continue
    t = tqdm.tqdm(iter(letters), leave=True, total=len(letters), desc='----Letter')
    for _, letter in enumerate(t):
        # if letter != LETTER and just_one_sample:
        #     continue
        for number in sorted(os.listdir(os.path.join(path_to_pointnet_data, data_split, letter))):
            # if number != NUMBER and just_one_sample:
            #     continue
            # if letter == "B" and int(number) < 304:
            #     continue
            path_to_pointnet_data_sequence = os.path.join(path_to_pointnet_data, data_split, letter, number)
            path_to_pointnet_data_features_sequence = os.path.join(path_to_pointnet_data_features, data_split, letter, number)
            create_dir(path_to_pointnet_data_features_sequence)

            for sample in sorted(os.listdir(os.path.join(path_to_pointnet_data_sequence))):
                path_to_pointnet_sample = os.path.join(path_to_pointnet_data_sequence, sample)
                path_to_pointnet_feature_sample = os.path.join(path_to_pointnet_data_features_sequence, sample)

                pointnet_data = np.load(path_to_pointnet_sample)
                # print(path_to_pointnet_feature_sample)
                #
                np.savez(path_to_pointnet_feature_sample,
                         voxels_features=pointnet_data['voxels_features'],
                         voxels_coords=pointnet_data['voxels_coords'],
                         allow_pickle=False)

max_stars_repo_name: pablorpalafox/voxflownet

max_issues_repo_name: pablopalafox/voxflownet

max_forks_repo_name: pablopalafox/voxflownet


text:
from models.few_shot.base import FewShotModel

import torch
import torch.nn as nn
import torch.nn.functional as F
from torch.optim import Optimizer
from torch.cuda.amp import autocast, GradScaler

import numpy as np

from typing import Callable

def fit_handle(
    model: nn,
    optimizer: Optimizer,
    scaler: GradScaler,
    loss_fn: Callable,
    mixup: bool = False
    ):
    # 返回core函数:
    def core(
        x: torch.Tensor,
        y: torch.Tensor,
        prefix: str = 'train_'
        ):
        if prefix == 'train_':
            model.train()

            with autocast():
                if mixup:
                    lambda_ = np.random.beta(0.2, 0.2)
                    index = torch.randperm(x.size(0)).cuda(non_blocking=True)
                    mix_x = lambda_ * x + (1 - lambda_) * x[index, :]

                    y_a, y_b = y, y[index]

                    logits, reg_logits = model(mix_x, prefix)

                    loss = lambda_ * loss_fn(logits, y_a) + (1 - lambda_) * loss_fn(logits, y_b)
                else:
                    logits, reg_logits = model(x, prefix)

                    loss = loss_fn(logits, y)

            # Take gradient step
            optimizer.zero_grad()
            scaler.scale(loss).backward()
            scaler.step(optimizer)
            scaler.update()

        else:
            model.eval()
            with autocast():
                logits, reg_logits = model(x, prefix)
                loss = loss_fn(logits, y)

        return logits, reg_logits, loss, y
    return core

class LinearClassifier(FewShotModel):
    def __init__(self, args, num_classes):
        super().__init__(args)
        # print("别忘了设置num_classes")
        self.v_cls = nn.Linear(1024*2, 500)

    def forward(self, x, prefix, get_feature=False):
        logits = self.encoder(x)
        if get_feature:
            logits = self.v_cls(logits).mean(1)

        return logits, None

max_stars_repo_name: ZhangYikaii/Proto-CAT

max_issues_repo_name: ZhangYikaii/Proto-CAT

max_forks_repo_name: ZhangYikaii/Proto-CAT


text:
#!/usr/bin/env python2
from __future__ import print_function
import roslib
import sys
import rospy
import csv
import numpy as np
import datetime
import time
from geometry_msgs.msg import Twist
from dse_msgs.msg import PoseMarkers
from std_msgs.msg import Float64MultiArray
from std_msgs.msg import MultiArrayLayout
from std_msgs.msg import MultiArrayDimension
from dse_msgs.msg import InfFilterPartials
from dse_msgs.msg import InfFilterResults
from scipy.spatial.transform import Rotation as R

import dse_lib
import dse_constants
roslib.load_manifest('dse_simulation')

class aruco_test:

    # Set up initial variables
    # Pass in the ID of this agent and the state dimension (6 or 12)
    def __init__(self):

        # # Get parameters from launch file
        # self.ros_prefix = rospy.get_param('~prefix')
        # if len(self.ros_prefix) != 0 and self.ros_prefix[0] != '/':
        #     self.ros_prefix = '/' + self.ros_prefix
        # self.camera_limits = rospy.get_param('~camera_limits')
        # # [dist_min, dist_max, horiz_fov, vert_fov]
        # self.tag_limit = rospy.get_param('~tag_limit')
        # # tag rotation at which the tag no longer detects
        # self.tag_size = rospy.get_param('~tag_size')
        # # height of the tag

        self.ros_prefix = '/tb3_0'
        self.camera_limits = [0.5, 1, 1.085595, 1.085595*480/640]
        self.tag_limit = 0.785398
        self.tag_size = 0.1*1.5

        # Define publishers and subscribers
        # Subscribe to the pose output from the camera
        self.pose_sub = rospy.Subscriber(self.ros_prefix + "/dse/pose_markers", PoseMarkers, self.camera_callback)
        self.num_results = 0
        self.results = []

    # When the direct estimator or consensus returns the combined information variables
    def camera_callback(self, data):
        self.num_results += 1
        result = np.zeros((10))
        result[0:6] = dse_lib.state_from_pose_array(data.pose_array, 12, 6)[0:6, 0]
        result[6:10] = dse_lib.eul2quat(result[3:6, None])
        self.results.append(result)

    def collect_data(self):
        if self.num_results > 0:
            tmp = np.array(self.results)
            [x, y, z, y, p, r, qx, qy, qz, qw] = np.split(np.array(self.results), 10, axis=1)
            print('Success')
            self.num_results = 0
            self.results = []
        print('data')

def main(args):
    with open('data.csv', mode="w") as data_file:
        data_writer = csv.writer(data_file, delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL)
        rospy.init_node('aruco_test_node', anonymous=True)
        il = aruco_test()
        il.csv_writer = data_writer
        r = rospy.Rate(1)
        il.dt = 1 / 1
        try:
            while True:
                r.sleep()
                il.collect_data()
        except KeyboardInterrupt:
            print("Shutting down")

if __name__ == '__main__':
    main(sys.argv)

max_stars_repo_name: hfekrmandi/Autonomous-GNC-MAS

reason: import numpy,from scipy


text:
import numpy as np

from pyhsmm.models import _HMMGibbsSampling, _HMMEM, _HMMMeanField
from pyhsmm.internals.initial_state import UniformInitialState

from autoregressive.models import _ARMixin
from autoregressive.util import AR_striding

from pyslds.models import _SLDSGibbsMixin, _SLDSVBEMMixin, _SLDSMeanFieldMixin

from rslds.states import InputHMMStates, PGRecurrentSLDSStates, SoftmaxRecurrentSLDSStates
import rslds.transitions as transitions

### Input-driven HMMs
class _InputHMMMixin(object):

    # Subclasses must specify the type of transition model
    _trans_class = None

    # custom init method, just so we call custom input trans class stuff
    def __init__(self,
                 obs_distns,
                 D_in=0,
                 trans_distn=None, trans_params={},
                 init_state_distn=None, init_state_concentration=None, pi_0=None,
                 ):
        self.obs_distns = obs_distns
        self.states_list = []
        self.D_in = D_in

        # our trans class
        if trans_distn is None:
            self.trans_distn = self._trans_class(num_states=len(obs_distns),
                                                 covariate_dim=D_in,
                                                 **trans_params)
        else:
            self.trans_distn = trans_distn

        if init_state_distn is not None:
            if init_state_distn == 'uniform':
                self.init_state_distn = UniformInitialState(model=self)
            else:
                self.init_state_distn = init_state_distn
        else:
            self.init_state_distn = self._init_state_class(
                    model=self,
                    init_state_concentration=init_state_concentration,
                    pi_0=pi_0)

        self._clear_caches()

    # custom add_data - includes a covariates arg
    def add_data(self, data, covariates=None, **kwargs):
        # NOTE! Our convention is that covariates[t] drives the
        # NOTE! transition matrix going into time t. However, for
        # NOTE! implementation purposes, it is easier if these inputs
        # NOTE! are lagged so that covariates[t] drives the input to
        # NOTE! z_{t+1}. Then, we only have T-1 inputs for the T-1
        # NOTE! transition matrices in the heterogeneous model.

        # Offset the covariates by one so that
        # the inputs at time {t-1} determine the transition matrix
        # from z_{t-1} to z_{t}.
        offset_covariates = covariates[1:]
        self.states_list.append(
                self._states_class(
                    model=self, data=data,
                    covariates=offset_covariates, **kwargs))

    def generate(self, T=100, covariates=None, keep=True):
        if covariates is None:
            covariates = np.zeros((T, self.D_in))
        else:
            assert covariates.ndim == 2 and \
                   covariates.shape[0] == T
        s = self._states_class(model=self, covariates=covariates[1:], T=T, initialize_from_prior=True)
        data = self._generate_obs(s)
        if keep:
            self.states_list.append(s)
        return (data, covariates), s.stateseq

    def resample_trans_distn(self):
        self.trans_distn.resample(
            stateseqs=[s.stateseq for s in self.states_list],
            covseqs=[s.covariates for s in self.states_list],
        )
        self._clear_caches()

class PGInputHMM(_InputHMMMixin, _HMMGibbsSampling):
    _trans_class = transitions.InputHMMTransitions
    _states_class = InputHMMStates

class PGInputOnlyHMM(PGInputHMM):
    _trans_class = transitions.InputOnlyHMMTransitions

class PGStickyInputOnlyHMM(PGInputHMM):
    _trans_class = transitions.StickyInputOnlyHMMTransitions

class SoftmaxInputHMM(_InputHMMMixin, _HMMGibbsSampling, _HMMEM):
    _trans_class = transitions.SoftmaxInputHMMTransitions
    _states_class = InputHMMStates

    ## EM
    def _M_step_trans_distn(self):
        zs = [s.expected_states.argmax(1).astype(np.int32) for s in self.states_list]
        xs = [s.covariates for s in self.states_list]
        xs = [np.row_stack([x, np.zeros(x.shape[1])]) for x in xs]
        self.trans_distn.initialize_with_logistic_regression(zs, xs)

class SoftmaxInputOnlyHMM(SoftmaxInputHMM):
    _trans_class = transitions.SoftmaxInputOnlyHMMTransitions

### ARHMM's
class _InputARHMMMixin(_InputHMMMixin, _ARMixin):

    def add_data(self, data, covariates=None, strided=False, **kwargs):
        if covariates is None:
            covariates = np.zeros((data.shape[0], 0))

        strided_data = AR_striding(data,self.nlags) if not strided else data
        lagged_covariates = covariates[self.nlags:]
        assert strided_data.shape[0] == lagged_covariates.shape[0]

        # Pass to InputHMM
        super(_InputARHMMMixin, self).add_data(data=strided_data,
                                               covariates=lagged_covariates,
                                               **kwargs)

class PGInputARHMM(_InputARHMMMixin, _HMMGibbsSampling):
    _trans_class = transitions.InputHMMTransitions
    _states_class = InputHMMStates

class PGInputOnlyARHMM(PGInputARHMM):
    _trans_class = transitions.InputOnlyHMMTransitions

class PGStickyInputOnlyARHMM(PGInputARHMM):
    _trans_class = transitions.StickyInputOnlyHMMTransitions

class SoftmaxInputARHMM(_InputARHMMMixin, _HMMGibbsSampling, _HMMEM):
    _trans_class = transitions.SoftmaxInputHMMTransitions
    _states_class = InputHMMStates

class SoftmaxInputOnlyARHMM(SoftmaxInputARHMM):
    _trans_class = transitions.SoftmaxInputOnlyHMMTransitions

### Recurrent ARHMM's
class _RecurrentARHMMMixin(_InputARHMMMixin):
    """
    In the "recurrent" version, the data also serve as covariates.
    """
    def add_data(self, data, covariates=None, strided=False, **kwargs):
        # Remember that the covariates[t] drives the transition probabilities p(z[t] | ...)
        # under our convention for add_data.
        T = data.shape[0]
        if covariates is None:
            covariates = np.zeros((T, 0))
        else:
            assert covariates.shape[0] == T

        # Combine the lagged data and the given covariates
        covariates = np.column_stack((
            np.row_stack((np.zeros(self.D), data[:-1])),
            covariates))

        super(_RecurrentARHMMMixin, self).add_data(data, covariates=covariates, **kwargs)

    def generate(self, T=100, keep=True, init_data=None, covariates=None, with_noise=True):
        from pybasicbayes.util.stats import sample_discrete
        # Generate from the prior and raise exception if unstable
        K, n = self.num_states, self.D

        # Prepare the covariates
        if covariates is None:
            covariates = np.zeros((T, 0))
        else:
            assert covariates.shape[0] == T

        # Initialize discrete state sequence
        pi_0 = self.init_state_distn.pi_0
        dss = np.empty(T, dtype=np.int32)
        dss[0] = sample_discrete(pi_0.ravel())

        data = np.empty((T, n), dtype='double')
        if init_data is None:
            data[0] = np.random.randn(n)
        else:
            data[0] = init_data

        for t in range(1, T):
            # Sample discrete state given previous continuous state and covariates
            cov_t = np.column_stack((data[t-1:t], covariates[t]))
            A = self.trans_distn.get_trans_matrices(cov_t)[0]
            dss[t] = sample_discrete(A[dss[t-1], :])

            # Sample continuous state given current discrete state
            if with_noise:
                data[t] = self.obs_distns[dss[t]].rvs(cov_t, return_xy=False)
            else:
                data[t] = self.obs_distns[dss[t]].predict(cov_t)

            assert np.all(np.isfinite(data[t])), "RARHMM appears to be unstable!"

        # TODO:
        # if keep:
        #     ...

        return data, dss

class PGRecurrentARHMM(_RecurrentARHMMMixin, _HMMGibbsSampling):
    _trans_class = transitions.InputHMMTransitions
    _states_class = InputHMMStates

class PGRecurrentOnlyARHMM(PGRecurrentARHMM):
    _trans_class = transitions.InputOnlyHMMTransitions

class PGStickyRecurrentOnlyARHMM(PGRecurrentARHMM):
    _trans_class = transitions.StickyInputOnlyHMMTransitions

class SoftmaxRecurrentARHMM(_RecurrentARHMMMixin, _HMMGibbsSampling, _HMMEM):
    _trans_class = transitions.SoftmaxInputHMMTransitions
    _states_class = InputHMMStates

class SoftmaxRecurrentOnlyARHMM(SoftmaxRecurrentARHMM):
    _trans_class = transitions.SoftmaxInputOnlyHMMTransitions

### Stick-breaking transition models with Pólya-gamma augmentation
class _RecurrentSLDSBase(object):
    def __init__(self, dynamics_distns, emission_distns, init_dynamics_distns,
                 fixed_emission=False, **kwargs):

        self.fixed_emission = fixed_emission

        # This class must always be used in conjunction with an SLDS class
        super(_RecurrentSLDSBase, self).__init__(
            dynamics_distns, emission_distns, init_dynamics_distns,
            D_in=dynamics_distns[0].D_out, **kwargs)

    def add_data(self, data, **kwargs):
        self.states_list.append(
                self._states_class(model=self, data=data, **kwargs))

class PGRecurrentSLDS(_RecurrentSLDSBase, _SLDSGibbsMixin, PGInputHMM):

    _states_class = PGRecurrentSLDSStates
    _trans_class = transitions.InputHMMTransitions

    def resample_trans_distn(self):
        # Include the auxiliary variables used for state resampling
        self.trans_distn.resample(
            stateseqs=[s.stateseq for s in self.states_list],
            covseqs=[s.covariates for s in self.states_list],
            omegas=[s.trans_omegas for s in self.states_list]
        )
        self._clear_caches()

    def resample_emission_distns(self):
        if self.fixed_emission:
            return
        super(PGRecurrentSLDS, self).resample_emission_distns()

class StickyPGRecurrentSLDS(PGRecurrentSLDS):
    _trans_class = transitions.StickyInputHMMTransitions

class PGRecurrentOnlySLDS(PGRecurrentSLDS):
    _trans_class = transitions.InputOnlyHMMTransitions

class StickyPGRecurrentOnlySLDS(PGRecurrentSLDS):
    _trans_class = transitions.StickyInputOnlyHMMTransitions

### Tree structured models
class TreeStructuredPGRecurrentSLDS(PGRecurrentSLDS):
    """
    - Pass in a TreeStructuredDynamics object
    - Exposed self.dynamics_distns as a pointer to dynamics.regressions
    - Change call to resample_dynamics_distns to pass appropriate inputs
    """
    def __init__(self, hierarchical_dynamics_distn, emission_distns, init_dynamics_distns,
                 fixed_emission=False, **kwargs):
        # Call the super constructor with the hierarchical dynamics regressions objects
        self.hierarchical_dynamics_distn = hierarchical_dynamics_distn
        dynamics_distns = hierarchical_dynamics_distn.regressions
        super(TreeStructuredPGRecurrentSLDS, self).__init__(
            dynamics_distns, emission_distns, init_dynamics_distns,
            fixed_emission=fixed_emission,
            **kwargs)

    def resample_dynamics_distns(self):
        zs = [s.stateseq[:-1] for s in self.states_list]
        xs = [np.hstack((s.gaussian_states[:-1], s.inputs[:-1]))
              for s in self.states_list]
        ys = [s.gaussian_states[1:] for s in self.states_list]
        self.hierarchical_dynamics_distn.resample(list(zip(xs, ys, zs)))
        self._clear_caches()

### Softmax transition models with variational inference
class SoftmaxRecurrentSLDS(_RecurrentSLDSBase, _SLDSMeanFieldMixin, _SLDSVBEMMixin, SoftmaxInputHMM):
    _states_class = SoftmaxRecurrentSLDSStates
    _trans_class = transitions.SoftmaxInputHMMTransitions

    def _M_step_trans_distn(self):
        stack_tuples = lambda lst: list(map(lambda xs: np.concatenate(xs, axis=0), zip(*lst)))
        self.trans_distn.max_likelihood(
            stats=stack_tuples([s.E_trans_stats for s in self.states_list]))

    def meanfield_update_trans_distn(self):
        # Include the auxiliary variables of the lower bound
        stack_tuples = lambda lst: list(map(lambda xs: np.concatenate(xs, axis=0), zip(*lst)))
        self.trans_distn.meanfieldupdate(
            stats=stack_tuples([s.E_trans_stats for s in self.states_list]))

    def _init_mf_from_gibbs(self):
        self.trans_distn._initialize_mean_field()
        super(SoftmaxRecurrentSLDS, self)._init_mf_from_gibbs()

    def initialize_transitions_from_gibbs(self):
        self.trans_distn.initialize_with_logistic_regression(
            [s.stateseq for s in self.states_list],
            [s.gaussian_states for s in self.states_list])

    def meanfield_update_parameters(self):
        self.meanfield_update_init_dynamics_distns()
        self.meanfield_update_dynamics_distns()
        self.meanfield_update_emission_distns()
        super(SoftmaxRecurrentSLDS, self).meanfield_update_parameters()

class SoftmaxRecurrentOnlySLDS(SoftmaxRecurrentSLDS):
    _trans_class = transitions.SoftmaxInputOnlyHMMTransitions

max_stars_repo_name: nitinshyamk/recurrent-slds

max_issues_repo_name: nitinshyamk/recurrent-slds

max_forks_repo_name: nitinshyamk/recurrent-slds


text:
# JUST implementing an edge queu to do LPA*
import sys
sys.path.append("../")
import code

import Visualizer
import time
import heapq
import numpy as np
import random
import matplotlib.pyplot as plt
plt.style.use("seaborn-dark")
inf = np.Inf

class State():
    def __init__(self):
        self.x = 5
        self.y = 5
        self.g = inf
        self.h = inf
        self.gT = inf

        self.rhs = inf

class Edge():
    def __init__(self):
        self.source_state = State()
        self.target_state = State()
        self.edgeCost = inf

class LAST:
    def __init__(self):
        # stuff for the environmnet
        self.xMin = 0.0
        self.xMax = 10.0

        self.yMin = 0.0
        self.yMax = 10.0

        self.start = State()
        self.start.x = 2
        self.start.y = 4

        self.goal = State()
        self.goal.x = 15
        self.goal.y = 12

        # map stuff
        self.obs = np.array([])
        # stuff for sampling
        self.m = 100
        self.Xsamples = {}

        # rgg stuff
        self.r = 2.0
        self.V = {}
        self.E = {}

        # state appending
        self.stateId = 0

    def calculate_L2(self, x1, y1, x2, y2):
        return np.sqrt((x2-x1)**2 + (y2-y1)**2)

    def calcDist(self, state1, state2):
        x1 = state1.x
        y1 = state1.y
        x2 = state2.x
        y2 = state2.y
        return self.calculate_L2(x1,y1,x2,y2)

    def readEnvironment(self, envFile, updated = False):
        A = []
        f = open(envFile)
        for x in f:
            print(x.rstrip("\n"))
            A.append(x.rstrip("\n"))
        print(A)

        self.xMax = int(A[0])
        self.yMax = int(A[1])
        print(self.xMax)
        print(self.yMax)
        self.obs = np.zeros((self.xMax, self.yMax))
        for j in range(self.yMax):
            for i in range(self.xMax):
                if A[2+j][i] == "-":
                    self.obs[j][i] = 0
                else:
                    self.obs[j][i] = 1

        if updated == False:
            self.obs1 = self.obs
            print("self.obs")
            print(self.obs)
            # our program has an
            self.start.x = float(A[2+self.yMax])
            self.start.y = float(A[3+self.yMax])
            self.goal.x = float(A[4+self.yMax])
            self.goal.y = float(A[5+self.yMax])

            print(self.start.x)
            print(self.start.y)
            print(self.goal.x)
            print(self.goal.y)
            self.start.gHat = 0
            self.start.hHat = self.calcDist(self.start,self.goal)
            self.start.fHat = self.start.gHat + self.start.hHat

            self.goal.gHat = self.calcDist(self.start, self.goal)
            self.goal.hHat = 0
            self.goal.fHat = self.goal.gHat + self.goal.hHat
            print("ENVIRONMENT READING fHat goal", self.start.fHat)
            print("ENVIRONMENT READING fHat goal", self.goal.fHat)
        else:
            print("I SET SELF.OBS 2")
            self.obs2 = self.obs

    def Sample(self):
        # add self.m number of valid samples
        #self.dbgSampleCount +=1

        # something to break out of the directly connected case
        i = 0
        ##print("DEBUG OUTPUT FOR def Sample(self):")
        ##print("self.m", self.m)
        ##print("self.c", self.c)
        while (i < self.m):
            xRand = random.uniform(self.xMin, self.xMax)
            yRand = random.uniform(self.yMin, self.yMax)
            xIdx = int(np.floor(xRand))
            yIdx = int(np.floor(self.yMax - yRand))

            # i'm pretty sure its supposed to sample from around where you are in the search
            tmpG = self.calculate_L2(xRand, yRand, self.start.x, self.start.y)
            tmpH = self.calculate_L2(xRand, yRand, self.goal.x, self.goal.y)

            ##print("xrand: " + str(xRand) + " yrand: " + str(yRand))
            ##print("tmpG " + str(tmpG) + " tmpH " + str(tmpH) )
            ##print("tmpH + tmpH = ", tmpG+ tmpH)
            ##print("goal.gT",self.goal.gT)
            if (tmpG+tmpH) < self.goal.gT:
                ##print("adding samples into Xsamples")
                # this sometimes causes an index error
                if self.obs[yIdx][xIdx] == 0:
                    stateToAdd = State()
                    stateToAdd.x = xRand
                    stateToAdd.y = yRand
                    stateToAdd.gHat = tmpG
                    stateToAdd.hHat = tmpH
                    stateToAdd.fHat = tmpG + tmpH
                    self.Xsamples[stateToAdd] = stateToAdd
                    i+=1
            ##print("length of Xsamples in Samples", len(self.Xsamples))
        return self.Xsamples

    def collisionCheck(self, Vm, Xm):
        #self.dbgCollisionCheckCount += 1
        # returns true if there is a collision
        x1 = Vm.x
        y1 = Vm.y
        x2 = Xm.x
        y2 = Xm.y

        # get the equation of the line
        m = (Xm.y - Vm.y)/(Xm.x - Vm.x)
        b = y1 - m*x1
        ##print("slope m = ",m)
        ##print("intercept b = ", b)
        # 200 checks per unit in X
        numberOfChecks = 200
        checkY = int(abs(y2-y1)*numberOfChecks)
        checkX = int(abs(x2-x1)*numberOfChecks)
        if checkY > checkX:
            collCheck = checkY
        else:
            collCheck = checkX
        t = np.linspace(x1, x2,collCheck) #10,000 to avoid the diag
        ##print("len t", len(t))
        Y = m*t + b
        ##plt.plot(t,Y)
        ##plt.xlim((0,11))
        ##plt.ylim((0,11))
        #plt.show()
        coordinate_t = np.floor(t)
        #print(Y)
        #print(self.xMax)
        #print(self.yMax-Y)
        coordinate_Y = np.floor(self.yMax-Y)
        #print(coordinate_t)
        #print(coordinate_Y)
        #print("PRINTING OBSTACLES")
        #print(self.obs)
        #print("PRINTING OBSTACLES FINISHED")
        # i iterates over the
        for i in range(len(coordinate_t)):
            idxToObs_t = int(coordinate_t[i])
            idxToObs_Y = int(coordinate_Y[i])
            #print("idxToObs_t" + str(idxToObs_t) + " idxToObs_Y " + str(idxToObs_Y))
            #print("iterating over the obstacles")
            #print(self.obs[idxToObs_Y][idxToObs_t])
            if self.obs[idxToObs_Y][idxToObs_t] == 1:
                return 1
        return 0

    def makeExplicitRGG(self):
        #self.V = self.Xsamples
        self.Xsamples[self.start] = self.start
        self.Xsamples[self.goal] = self.goal

        for sample in self.Xsamples:
            sample.h = self.calcDist(sample, self.goal)

        self.Xsamples2 = self.Xsamples.copy()

        # start and goal stuff
        self.start.g = 0
        self.start.h = self.calcDist(self.start, self.goal)

        self.goal.h = 0
        self.goal.g = self.calcDist(self.start, self.goal)

        for i in self.Xsamples:
            for j in self.Xsamples2:
                dist = self.calcDist(i,j)
                #print(dist)
                if dist <= self.r and i != j:
                    edgeToAdd = Edge()
                    edgeToAdd.source_state = i
                    edgeToAdd.target_state = j
                    edgeToAdd.edgeCost = self.calcDist(i,j)
                    self.E[edgeToAdd] = edgeToAdd
        self.V = self.Xsamples.copy()

    def updateEdgeCosts(self):
        # to be called AFTER you load in second map
        print("UPDATING EDGE COSTS")
        changedEdges = []
        for E in self.E:
            tmpEdgeCost = E.edgeCost
            if self.collisionCheck(E.source_state, E.target_state) == 1:
                E.edgeCost = inf
            else:
                E.edgeCost = self.calcDist(E.source_state, E.target_state)

            if E.edgeCost != tmpEdgeCost:
                changedEdges.append(E)
        return changedEdges

    def Initialize(self):
        self.changedMap = [[ [False, False] for x in range(self.xMax)] for y in range(self.yMax)]
        self.U = []         #{01}, {04 i do somewhere else  }
        startState = self.V[self.start]
        startKey = self.CalculateKey(startState)
        self.stateId += 1
        heapq.heappush(self.U, (startKey,self.stateId, startState)) # {05}

    def CalculateKey(self, state):
        # put that shit right on the state, then this function is easy
        # so i need to be putting the g, rhs, and h values on the state when i update vertex
        # this assumes that that stuff is calculated on the state already when you go to look for it
        # and that might not be true
        g = state.g
        rhs = state.rhs
        h = state.h
        minItem = min(g,rhs)
        return [minItem + h, minItem]

    def UpdateVertex(self, edge, secondCase = False):
        # my s is his u
        s = edge.target_state
        root = edge.source_state

        print("root.x, root.y", root.x, root.y)
        print("s.x, s.y", s.x, s.y)

        rhsList = []
        cameFromIdxList = []
        cameFromCoordList = []
        fList = []
        preds = self.getPred(s)

        for pred in preds:
            #print(pred)
            #if pred.target_state !=  self.start:

            if self.collisionCheck(pred.source_state,pred.target_state) == 1:
                pred.edgeCost = inf
            tentativeRhs = pred.edgeCost + pred.source_state.g
            rhsList.append( tentativeRhs )
            cameFromIdxList.append(pred.source_state)

        if s != self.start:
            minRhs = min(rhsList)
            print("min Rhs", minRhs)
            print("from rhsList", rhsList)
            min_index = rhsList.index(minRhs)
            print("min index", min_index)
            print("cameFromIdxList", cameFromIdxList)

            s.rhs = minRhs
            s.f = s.rhs + s.h
            #s.g = inf
            s.cameFromIdx = cameFromIdxList[min_index]
            print(cameFromIdxList)

            #code.interact(local=locals())
            #s.cameFromCoordList = cameFromCoordList[min_index]
        else:   #s must be the start
            s.rhs = 0

        # this line once stopped me from terminating
        for keyStatePair in self.U:
            #print("keystatepair", keyStatePair)
            #print("keystatepair[2]", keyStatePair[2])
            if keyStatePair[2] == s:
                self.U.remove(keyStatePair)
        print("s.g", s.g)
        print("s.rhs", s.rhs)

        if s.g != s.rhs:
            self.stateId+=1
            heapq.heappush(self.U, (self.CalculateKey(s), self.stateId, s) )
            print("enqueued onto U")
        print("finished update vertex")

    def getPred(self,u):
        self.succs = []
        for key, edge in self.E.items():
            if edge.target_state == u:
                self.succs.append(edge)
                #print("succs.edgeCost", edge.edgeCost)
        #print("self.succs", self.succs)
        return self.succs

    def getSucc(self,u):
        # given a state, return the successors
        self.succs = []
        for key, edge in self.E.items():
            # only print these for debugging it makes everythin really slow
            #print("key", key)
            #print("edge", edge)
            if edge.source_state == u:
                self.succs.append(edge)
                #print("succs.edgeCost", edge.edgeCost)
        #print("self.succs", self.succs)
        return self.succs

    def bestU(self):
        # return the key in lex order unless the set is emtpy then return [inf, inf]
        if len(self.U) > 0:
            return self.U[0][0]
        else:
            return [inf,inf]

    def ComputeShortestPath(self):
        #print("self.U", self.U)
        #print("self.U[0]", self.U[0])
        #while self.U[0][0] < self.CalculateKey(self.V[(self.goal.iX, self.goal.iY)]) or (self.V[(self.goal.iX, self.goal.iY)].rhs != self.V[(self.goal.iX, self.goal.iY)].g):
        while self.bestU() < self.CalculateKey(self.V[self.goal]) or (self.V[self.goal].rhs != self.V[self.goal].g):
            #print("self.U", self.U)
            u = heapq.heappop(self.U)[2]
            #print("u", u)
            if u.g > u.rhs:
                print("first case")
                u.g = u.rhs
                self.succs = self.getSucc(u)
                #print("self.succs", self.succs)
                for edge in self.succs:
                    self.UpdateVertex(edge, True)
                    #self.motionE[(edge.source_state, edge.target_state)] = edge
            else:
                print("second case")
                if u != self.start:
                    u.g = inf
                self.succs = self.getSucc(u)

                #edgeU = Edge()
                #edgeU.source_state = u
                #edgeU.target_state = u      # {union of {16}}
                #edgeU.edgeCost = 0
                # actually appending this loops the algorithm forever
                # self.succs.append(edgeU)       # lets worry about this later
                for edge in self.succs:
                    self.UpdateVertex(edge, True)
                    #self.motionE[(edge.source_state, edge.target_state)] = edge

    def Main(self):
        Xsamples = self.Sample()
        print("Xsamples", Xsamples)
        self.makeExplicitRGG()

        self.Initialize()

        tcsp1_start = time.time()
        self.ComputeShortestPath()
        tcsp1_end = time.time()
        tcsp1 = tcsp1_end - tcsp1_start

        self.solution1 = []
        self.solution2 = []

        stateOfInterest = self.goal
        self.solution1.append(self.goal)
        timeOut = 0
        while stateOfInterest != self.start :
            if timeOut >500:
                break
            self.solution1.append(stateOfInterest.cameFromIdx)
            stateOfInterest = self.V[stateOfInterest.cameFromIdx]
            print("i am stuck in the first solution path :[")
            timeOut+=1
            print("timeOut", timeOut)
        print("solution path 1", self.solution1)

        ## REACT TO UPDATES
        self.readEnvironment(fileList[1], True)
        changedEdges = self.updateEdgeCosts()

        print(changedEdges)
        for changedEdge in changedEdges:
            self.UpdateVertex(changedEdge)

        tcsp2_start = time.time()
        self.ComputeShortestPath()
        tcsp2_end = time.time()
        tcsp2 = tcsp2_end - tcsp2_start

        stateOfInterest = self.goal
        self.solution2.append(self.goal)
        timeOut = 0
        while stateOfInterest != self.start :
            if timeOut >500:
                break
            self.solution2.append(stateOfInterest.cameFromIdx)
            stateOfInterest = self.V[stateOfInterest.cameFromIdx]
            print("i am stuck in the first solution path :[")
            timeOut+=1
            print("timeOut", timeOut)
        print("solution path 2", self.solution2)

        return self.solution1, self.solution2, changedEdges, tcsp1, tcsp2

if __name__ == "__main__":
    #random.seed(69) # my lucky number
    #random.seed(10)
    #fileList = ["../test_environments/grid_envs_changing/environment50_A_clear.txt", "../test_environments/grid_envs_changing/environment50_A_77.txt"]
    #fileList = ["../test_environments/grid_envs_changing/environment50_A_69.txt", "../test_environments/grid_envs_changing/environment50_A_77.txt"]
    #fileList = ["../test_environments/grid_envs_changing10/environment10_A_10.txt", "../test_environments/grid_envs_changing10/environment10_B_10.txt"]
    #fileList = ["../test_environments/grid_envs_changing10/environment10_B_10.txt", "../test_environments/grid_envs_changing10/environment10_A_10.txt"]
    #fileList = ["../test_environments/grid_envs_changing10/environment10_A_11.txt", "../test_environments/grid_envs_changing10/environment10_B_11.txt"]
    #fileList = ["../test_environments/grid_envs_changing10/environment10_B_11.txt", "../test_environments/grid_envs_changing10/environment10_A_11.txt"]
    #fileList = ["../test_environments/grid_envs_changing10/environment10_A_69.txt", "../test_environments/grid_envs_changing10/environment10_B_69.txt"]
    fileList = ["../test_environments/grid_envs_changing10/environment10_B_69.txt", "../test_environments/grid_envs_changing10/environment10_A_69.txt"]
    print("LAST CHANCE")
    L = LAST()

    L.readEnvironment(fileList[0], False)
    solution1, solution2, changedEdges, tcsp1, tcsp2 = L.Main()
    solution1X = []
    solution1Y = []
    for state in solution1:
        #print("statex, statey", state.x, state.y)
        solution1X.append(state.x)
        solution1Y.append(state.y)

    solution2X = []
    solution2Y = []
    for state in solution2:
        #print("statex, statey", state.x, state.y)
        solution2X.append(state.x)
        solution2Y.append(state.y)

    print("time for each results")
    print("tcsp1 = %s\n"%tcsp1)
    print("tcsp2 = %s\n"%tcsp2)
    gv = Visualizer.Visualizer()
    gv.plotEdges(L)
    gv.plotEnv(L)
    #gv.plotSolutionEdges(solution1)
    gv.lcPlotSolution(solution1X, solution1Y, "#420690", 5)
    gv.lcPlotSolution(solution2X, solution2Y, "#007fff", 3)
    gv.plotSamples(L)
    gv.labelLastChance(L)


text:
#!/usr/bin/env python
""" Fast convolution in the UV domain """
__author__ = "Wasim Raja"

import numpy as np
import astropy.units as units
import racs_tools.gaussft as gaussft

def convolve(image, old_beam, new_beam, dx, dy):
    """Convolve by X-ing in the Fourier domain.
        - convolution with Gaussian kernels only
        - no need for generation of a kernel image
        - direct computation of the FT of the kernel
        - dimension of FT = dimension of image

    Args:
        image (2D array): The image to be convolved.
        old_beam (radio_beam.Beam): Current image PSF.
        new_beam (radio_beam.Beam): Target image PSF.
        dx (float): Grid size in x in degrees (e.g. CDELT1)
        dy (float): Grid size in y in degrees (e.g. CDELT2)

    Returns:
        tuple: (convolved image, scaling factor)
    """
    nx = image.shape[0]
    ny = image.shape[1]

    # The coordinates in FT domain:
    u = np.fft.fftfreq(nx, d=dx.to(units.rad).value)
    v = np.fft.fftfreq(ny, d=dy.to(units.rad).value)

    g_final = np.zeros((nx, ny), dtype=float)
    [g_final, g_ratio] = gaussft.gaussft(bmin_in=old_beam.minor.to(units.deg).value,
                                         bmaj_in=old_beam.major.to(units.deg).value,
                                         bpa_in=old_beam.pa.to(units.deg).value,
                                         bmin=new_beam.minor.to(units.deg).value,
                                         bmaj=new_beam.major.to(units.deg).value,
                                         bpa=new_beam.pa.to(units.deg).value,
                                         u=u, v=v,
                                         nx=nx, ny=ny)
    # Perform the x-ing in the FT domain
    im_f = np.fft.fft2(image)

    # Now convolve with the desired Gaussian:
    M = np.multiply(im_f, g_final)
    im_conv = np.fft.ifft2(M)
    im_conv = np.real(im_conv)

    # print("factor: %f" % g_ratio)
    # print("dx: %s" % dx)
    # print("dy: %s" % dy)
    # tmp = old_beam.minor.to(units.deg).value
    # print("bMaj psf: %f , %f" % (tmp,tmp*3600))
    # tmp = bmin_in
    # print("bMaj psf: %f , %f" % (tmp,tmp*3600.0))
    # tmp = bpa_in
    # print("bPA psf: %f " % tmp)
    # tmp = bmaj
    # print("bMaj desired: %f, %f" % (tmp,tmp*3600.0))
    # tmp = bmin
    # print("bMin desired: %f, %f" % (tmp,tmp*3600.0))
    # tmp = bpa
    # print("bPA desired: %f " % tmp)
    return im_conv, g_ratio

max_stars_repo_name: dlakaplan/RACS-tools

max_issues_repo_name: dlakaplan/RACS-tools

max_forks_repo_name: dlakaplan/RACS-tools

reason: import numpy,import astropy


text:
import nltk
try:
    nltk.data.find('punkt')
    nltk.data.find('stopwords')

except LookupError:
    nltk.download('punkt')
    nltk.download('stopwords')
from keras.preprocessing.sequence import pad_sequences
from keras.models import load_model
from nltk import word_tokenize
from nltk.corpus import stopwords
import re
import string
import pickle
import numpy as np

class SentimentAnalysis(object):
    def __init__(self):
        self.stoplist = stopwords.words('english')
        self.model_file = self.load_model_file()
        self.MAX_SEQUENCE_LENGTH = 50

    def predict_phrase(self, phrase):
        pos, neg = 0, 0
        self.tokenizer_file = self.load_tokenize_pickle()
        data = self.clean_phrase(phrase)
        tokenize_data = self.tokenize_phrase(data)
        data_sorted = sorted(list(set(tokenize_data)))
        self.tokenizer_file.fit_on_texts(data_sorted)
        training_sequences = self.tokenizer_file.texts_to_sequences(data_sorted)

        final_data = pad_sequences(training_sequences, maxlen=self.MAX_SEQUENCE_LENGTH)

        prediction = self.model_file.predict(final_data)
        analysis_ave, pos_ave, neg_ave = self.analysis_by_average(prediction)
        analysis_max = self.analysis_by_max(prediction)
        if analysis_max == 1:
            pos += 1
        else:
            neg += 1
        if analysis_ave == 1:
            pos += 1
        else:
            neg += 1
        print("POS: ", pos, " NEG: ", neg, " MAX: ", analysis_max, " AVE: ", analysis_ave)
        print("POS_AVE: ", pos_ave)
        print("NEG_AVE: ", neg_ave)
        if pos > neg:
            return "Positive", pos_ave, neg_ave
        else:
            return "Negative", pos_ave, neg_ave

    def analysis_by_max(self, prediction):
        max_score_idx = np.argmax(prediction)
        a = prediction.reshape(-1, len(prediction) * 2)[0]
        print("SHAPE: ", a)
        print("MAX: ", max_score_idx)
        tmp = {}
        i = 0
        for c in a:
            if i % 2 == 0:
                tmp[i] = 0
            else:
                tmp[i] = 1
            i += 1
        print("TMP : ", tmp)
        return tmp[max_score_idx]

    def analysis_by_average(self, prediction):
        pos_sum = 0
        neg_sum = 0
        for col in prediction:
            neg_sum += col[0]
            pos_sum += col[1]
        ave_pos = (pos_sum / len(prediction))
        ave_neg = (neg_sum / len(prediction))
        if ave_pos > ave_neg:
            return 1, ave_pos, ave_neg
        else:
            return 0, ave_pos, ave_neg

    def clean_phrase(self, phrase):
        cleaned_html = self.remove_html(phrase)
        remove_punc = self.remove_punct(cleaned_html)
        cleaned_final = self.clean_final(remove_punc)
        return cleaned_final

    def tokenize_phrase(self, phrase):
        words_tokenize = word_tokenize(phrase)
        filtered_words = [self.remove_stop_words(sen) for sen in words_tokenize]
        filtered_words = filter(None, filtered_words)
        result = [' '.join(sen) for sen in filtered_words]
        return result

    def load_tokenize_pickle(self):
        tokenizer_loaded = None
        try:
            with open('model/tokenizer.pickle', 'rb') as handle:
                tokenizer_loaded = pickle.load(handle)
                return tokenizer_loaded
        except FileNotFoundError:
            print("Missing tokenizer pickle file!")

    def load_model_file(self):
        try:
            model = load_model('model/rcnn-lstm-model.h5')
            return model
        except FileNotFoundError:
            print("File model not found!")

    def remove_stop_words(self, tokens):
        if tokens not in self.stoplist:
            return tokens

    def remove_html(self, text):
        # clean=re.compile('<.*?>')
        cleantext = re.sub('<[^<]+?>', '', text)
        return cleantext

    def remove_punct(self, text):
        text = text.lower()
        text = re.sub('\[.*?\]', '', text)
        text = re.sub('[%s]' % re.escape(string.punctuation), '', text)
        text = re.sub('\w*\d\w*', '', text)
        return text

    def clean_final(self, text):
        text = re.sub('[''"",,,]', '', text)
        text = re.sub('\n', '', text)
        return text

max_stars_repo_name: Senpaixyz/RNN-sentiment-analysis

max_issues_repo_name: Senpaixyz/RNN-sentiment-analysis

max_forks_repo_name: Senpaixyz/RNN-sentiment-analysis