update

.gitattributes

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,11 @@
+
+.git/
+.idea/
+
+/data/
+/dotenv/
+/logs/
+/trained_models
+/temp/
+
+**/__pycache__/

Dockerfile

@@ -0,0 +1,15 @@
+FROM python:3.10
+
+RUN useradd -m -u 1000 user
+
+WORKDIR /app
+
+COPY --chown=user ./requirements.txt requirements.txt
+
+RUN pip install --no-cache-dir --upgrade -r requirements.txt
+
+COPY --chown=user . /app
+
+RUN bash install.sh --stage 0 --stop_stage 2
+
+CMD ["python3", "main.py"]

README.md

@@ -0,0 +1,11 @@
+---
+title: Part Of Speech
+emoji: 😻
+colorFrom: blue
+colorTo: blue
+sdk: docker
+pinned: false
+license: apache-2.0
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

examples/tutorial_pyltp/README.md

@@ -0,0 +1,10 @@
+## pyltp
+
+```text
+工程路径: 
+https://github.com/HuangFJ/pyltp
+
+模型文件: 
+https://ltp.ai/download.html
+
+```

examples/tutorial_pyltp/parser.py

@@ -0,0 +1,102 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+import argparse
+import os
+import time
+
+from project_settings import project_path
+
+os.environ['NLTK_DATA'] = (project_path / "data/nltk_data").as_posix()
+
+from nltk import DependencyGraph
+from pyltp import Segmentor
+from pyltp import Parser
+from pyltp import Postagger
+from pyltp import SementicRoleLabeller
+
+
+def get_args():
+    parser = argparse.ArgumentParser()
+
+    parser.add_argument(
+        "--text",
+        default="元芳你怎么看？我就趴窗口上看呗！",
+        # default="集中竞价的方式回购公司股份",
+        # default="，全中国都是我的",
+        # default="可以啊可以",
+        # default="我们是免费办理的, 不会收取任何手续费和服务费, 随借随还, 可以留个备用,您看可以吗？",
+        # default="。啊不用不用挂断 你这个昨天来过电话 你哪有打不通",
+        # default="利息怎么算",
+        type=str
+    )
+    parser.add_argument(
+        "--ltp_data_dir",
+        default=(project_path / "data/pyltp_models/ltp_data_v3.4.0").as_posix(),
+        type=str
+    )
+    args = parser.parse_args()
+    return args
+
+
+def main():
+    """
+    句法分析, 并画出句法树
+
+    参考链接:
+    https://www.freesion.com/article/91401299576/
+    """
+    args = get_args()
+
+    cws_model_path = os.path.join(args.ltp_data_dir, 'cws.model')
+    pos_model_path = os.path.join(args.ltp_data_dir, 'pos.model')
+    parser_model_path = os.path.join(args.ltp_data_dir, 'parser.model')
+    srl_model_path = os.path.join(args.ltp_data_dir, 'pisrl_win.model')
+
+    segmentor = Segmentor(cws_model_path)
+    pos_tagger = Postagger(pos_model_path)
+    parser = Parser(parser_model_path)
+    srl_labeler = SementicRoleLabeller(srl_model_path)
+
+    time_begin = time.time()
+
+    words = segmentor.segment(args.text)
+    words_ = [word for word in words]
+    print(words_)
+
+    postags = pos_tagger.postag(words)
+    postags_ = [postag for postag in postags]
+    print(postags_)
+
+    arcs = parser.parse(words, postags)
+
+    cost = time.time() - time_begin
+    print("cost: {}".format(cost))
+
+    tree_str = ""
+    for word, postag, arc in zip(words, postags, arcs):
+        head = arc[0]
+        relation = arc[1]
+        if head == 0:
+            relation = "ROOT"
+
+        line = """\t{word}({relation}/{postag})\t{postag}\t{head}\t{relation}\n""".format(
+            word=word,
+            relation=relation,
+            postag=postag,
+            head=head,
+        )
+        tree_str += line
+
+    print(tree_str)
+    conlltree = DependencyGraph(tree_str=tree_str)
+    tree = conlltree.tree()
+    tree.draw()
+
+    segmentor.release()
+    pos_tagger.release()
+    parser.release()
+    return
+
+
+if __name__ == '__main__':
+    main()

examples/tutorial_pyltp/part_of_speech.py

@@ -0,0 +1,53 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+"""
+https://huggingface.co/LTP
+"""
+import argparse
+import os
+
+from pyltp import Segmentor
+from pyltp import Postagger
+
+from project_settings import project_path
+
+
+def get_args():
+    parser = argparse.ArgumentParser()
+
+    parser.add_argument(
+        "--text",
+        default="元芳你怎么看？我就趴窗口上看呗！",
+        type=str
+    )
+    parser.add_argument(
+        "--ltp_data_dir",
+        default=(project_path / "data/pyltp_models/ltp_data_v3.4.0").as_posix(),
+        type=str
+    )
+    args = parser.parse_args()
+    return args
+
+
+def main():
+    args = get_args()
+
+    cws_model_path = os.path.join(args.ltp_data_dir, "cws.model")
+    pos_model_path = os.path.join(args.ltp_data_dir, "pos.model")
+
+    segmentor = Segmentor(cws_model_path)
+    postagger = Postagger(pos_model_path)
+
+    words = segmentor.segment(args.text)
+    postags = postagger.postag(words)
+    print(words)
+    print(postags)
+
+    segmentor.release()
+    postagger.release()
+
+    return
+
+
+if __name__ == "__main__":
+    main()

examples/tutorial_pyltp/sentence_splitter.py

@@ -0,0 +1,29 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+import argparse
+
+from pyltp import SentenceSplitter
+
+
+def get_args():
+    parser = argparse.ArgumentParser()
+
+    parser.add_argument(
+        "--text",
+        default="元芳你怎么看？我就趴窗口上看呗！",
+        type=str
+    )
+    args = parser.parse_args()
+    return args
+
+
+def main():
+    args = get_args()
+
+    sent_list = SentenceSplitter.split(args.text)
+    print(sent_list)
+    return
+
+
+if __name__ == "__main__":
+    main()

examples/tutorial_pyltp/tokenization.py

@@ -0,0 +1,47 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+"""
+https://huggingface.co/LTP
+"""
+import argparse
+import os
+
+from pyltp import Segmentor
+
+from project_settings import project_path
+
+
+def get_args():
+    parser = argparse.ArgumentParser()
+
+    parser.add_argument(
+        "--text",
+        default="元芳你怎么看？我就趴窗口上看呗！",
+        type=str
+    )
+    parser.add_argument(
+        "--ltp_data_dir",
+        default=(project_path / "data/pyltp_models/ltp_data_v3.4.0").as_posix(),
+        type=str
+    )
+    args = parser.parse_args()
+    return args
+
+
+def main():
+    args = get_args()
+
+    cws_model_path = os.path.join(args.ltp_data_dir, "cws.model")
+
+    segmentor = Segmentor(cws_model_path)
+
+    words = segmentor.segment(args.text)
+    print(words)
+
+    segmentor.release()
+
+    return
+
+
+if __name__ == "__main__":
+    main()

install.sh

@@ -0,0 +1,84 @@
+#!/usr/bin/env bash
+
+# bash install.sh --stage 0 --stop_stage 2
+# bash install.sh --stage 1 --stop_stage 1
+
+verbose=true;
+stage=-1
+stop_stage=0
+
+
+# parse options
+while true; do
+  [ -z "${1:-}" ] && break;  # break if there are no arguments
+  case "$1" in
+    --*) name=$(echo "$1" | sed s/^--// | sed s/-/_/g);
+      eval '[ -z "${'"$name"'+xxx}" ]' && echo "$0: invalid option $1" 1>&2 && exit 1;
+      old_value="(eval echo \\$$name)";
+      if [ "${old_value}" == "true" ] || [ "${old_value}" == "false" ]; then
+        was_bool=true;
+      else
+        was_bool=false;
+      fi
+
+      # Set the variable to the right value-- the escaped quotes make it work if
+      # the option had spaces, like --cmd "queue.pl -sync y"
+      eval "${name}=\"$2\"";
+
+      # Check that Boolean-valued arguments are really Boolean.
+      if $was_bool && [[ "$2" != "true" && "$2" != "false" ]]; then
+        echo "$0: expected \"true\" or \"false\": $1 $2" 1>&2
+        exit 1;
+      fi
+      shift 2;
+      ;;
+
+    *) break;
+  esac
+done
+
+work_dir="$(pwd)"
+data_dir="$(pwd)/data"
+nltk_data_dir="${data_dir}/nltk_data"
+nltk_data_tokenizers_dir="${nltk_data_dir}/tokenizers"
+pyltp_models_dir="${data_dir}/pyltp_models"
+
+mkdir -p "${data_dir}"
+mkdir -p "${nltk_data_dir}"
+mkdir -p "${nltk_data_tokenizers_dir}"
+mkdir -p "${pyltp_models_dir}"
+
+if [ ${stage} -le 0 ] && [ ${stop_stage} -ge 0 ]; then
+  $verbose && echo "stage 0: download nltk data punkt"
+  cd "${nltk_data_tokenizers_dir}" || exit 1;
+
+  # https://www.nltk.org/nltk_data/
+  if [ ! -d "punkt" ]; then
+  # nltk==3.8.1
+    wget -c https://raw.githubusercontent.com/nltk/nltk_data/gh-pages/packages/tokenizers/punkt.zip
+    unzip punkt.zip
+    rm punkt.zip
+  fi
+
+  if [ ! -d "punkt_tab" ]; then
+    # nltk==3.8.2
+    wget -c https://raw.githubusercontent.com/nltk/nltk_data/gh-pages/packages/tokenizers/punkt_tab.zip
+    unzip punkt_tab.zip
+    rm punkt_tab.zip
+  fi
+fi
+
+
+if [ ${stage} -le 1 ] && [ ${stop_stage} -ge 1 ]; then
+  $verbose && echo "stage 0: download pyltp models"
+  cd "${pyltp_models_dir}" || exit 1;
+
+  # pyltp
+  # https://ltp.ai/download.html
+  if [ ! -e "resources.json" ]; then
+    wget -c http://model.scir.yunfutech.com/model/ltp_data_v3.4.0.zip
+    unzip ltp_data_v3.4.0.zip
+    rm ltp_data_v3.4.0.zip
+  fi
+
+fi

log.py

@@ -0,0 +1,110 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+import logging
+from logging.handlers import TimedRotatingFileHandler
+import os
+
+
+def setup(log_directory: str):
+    fmt = "%(asctime)s - %(name)s - %(levelname)s  %(filename)s:%(lineno)d >  %(message)s"
+
+    stream_handler = logging.StreamHandler()
+    stream_handler.setLevel(logging.INFO)
+    stream_handler.setFormatter(logging.Formatter(fmt))
+
+    # main
+    main_logger = logging.getLogger("main")
+    main_logger.addHandler(stream_handler)
+    main_info_file_handler = TimedRotatingFileHandler(
+        filename=os.path.join(log_directory, "main.log"),
+        encoding="utf-8",
+        when="midnight",
+        interval=1,
+        backupCount=30
+    )
+    main_info_file_handler.setLevel(logging.INFO)
+    main_info_file_handler.setFormatter(logging.Formatter(fmt))
+    main_logger.addHandler(main_info_file_handler)
+
+    # http
+    http_logger = logging.getLogger("http")
+    http_file_handler = TimedRotatingFileHandler(
+        filename=os.path.join(log_directory, "http.log"),
+        encoding='utf-8',
+        when="midnight",
+        interval=1,
+        backupCount=30
+    )
+    http_file_handler.setLevel(logging.DEBUG)
+    http_file_handler.setFormatter(logging.Formatter(fmt))
+    http_logger.addHandler(http_file_handler)
+
+    # api
+    api_logger = logging.getLogger("api")
+    api_file_handler = TimedRotatingFileHandler(
+        filename=os.path.join(log_directory, "api.log"),
+        encoding='utf-8',
+        when="midnight",
+        interval=1,
+        backupCount=30
+    )
+    api_file_handler.setLevel(logging.DEBUG)
+    api_file_handler.setFormatter(logging.Formatter(fmt))
+    api_logger.addHandler(api_file_handler)
+
+    # alarm
+    alarm_logger = logging.getLogger("alarm")
+    alarm_file_handler = TimedRotatingFileHandler(
+        filename=os.path.join(log_directory, "alarm.log"),
+        encoding="utf-8",
+        when="midnight",
+        interval=1,
+        backupCount=30
+    )
+    alarm_file_handler.setLevel(logging.DEBUG)
+    alarm_file_handler.setFormatter(logging.Formatter(fmt))
+    alarm_logger.addHandler(alarm_file_handler)
+
+    debug_file_handler = TimedRotatingFileHandler(
+        filename=os.path.join(log_directory, "debug.log"),
+        encoding="utf-8",
+        when="D",
+        interval=1,
+        backupCount=7
+    )
+    debug_file_handler.setLevel(logging.DEBUG)
+    debug_file_handler.setFormatter(logging.Formatter(fmt))
+
+    info_file_handler = TimedRotatingFileHandler(
+        filename=os.path.join(log_directory, "info.log"),
+        encoding="utf-8",
+        when="D",
+        interval=1,
+        backupCount=7
+    )
+    info_file_handler.setLevel(logging.INFO)
+    info_file_handler.setFormatter(logging.Formatter(fmt))
+
+    error_file_handler = TimedRotatingFileHandler(
+        filename=os.path.join(log_directory, "error.log"),
+        encoding="utf-8",
+        when="D",
+        interval=1,
+        backupCount=7
+    )
+    error_file_handler.setLevel(logging.ERROR)
+    error_file_handler.setFormatter(logging.Formatter(fmt))
+
+    logging.basicConfig(
+        level=logging.DEBUG,
+        datefmt="%a, %d %b %Y %H:%M:%S",
+        handlers=[
+            debug_file_handler,
+            info_file_handler,
+            error_file_handler,
+        ]
+    )
+
+
+if __name__ == "__main__":
+    pass

main.py

@@ -0,0 +1,132 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+import argparse
+import asyncio
+import logging
+import json
+import os
+import platform
+import time
+
+from project_settings import project_path
+
+os.environ["NLTK_DATA"] = (project_path / "data/nltk_data").as_posix()
+os.environ["LTP_DATA_DIR"] = (project_path / "data/pyltp_models/ltp_data_v3.4.0").as_posix()
+
+from project_settings import project_path, log_directory
+import log
+
+log.setup(log_directory=log_directory)
+
+import gradio as gr
+
+from toolbox.os.command import Command
+from toolbox.part_of_speech.part_of_speech import language_to_engines, engine_to_tagger, pos_parser
+
+main_logger = logging.getLogger("main")
+
+
+def get_args():
+    parser = argparse.ArgumentParser()
+
+    parser.add_argument(
+        "--pos_example_json_file",
+        default=(project_path / "pos_examples.json").as_posix(),
+        type=str
+    )
+    args = parser.parse_args()
+    return args
+
+
+def pos_tag(text: str, language: str, engine: str) -> str:
+    begin = time.time()
+
+    try:
+        words, postags = pos_parser(text, engine)
+        result = ""
+        for word, postag in zip(words, postags):
+            row = f"{word}/{postag}"
+            result += f"{row}\t"
+
+        time_cost = time.time() - begin
+        result += f"\n\ntime_cost: {round(time_cost, 4)}"
+        return result
+    except Exception as e:
+        result = f"{type(e)}\n{str(e)}"
+        return result
+
+
+def shell(cmd: str):
+    return Command.popen(cmd)
+
+
+def main():
+    args = get_args()
+
+    with open(args.pos_example_json_file, "r", encoding="utf-8") as f:
+        pos_examples: list = json.load(f)
+
+    def get_languages_by_engine(engine: str):
+        language_list = list()
+        for k, v in language_to_engines.items():
+            if engine in v:
+                language_list.append(k)
+        return gr.Dropdown(choices=language_list, value=language_list[0], label="language")
+
+    pos_language_choices = list(language_to_engines.keys())
+    pos_engine_choices = list(engine_to_tagger.keys())
+
+    # blocks
+    with gr.Blocks() as blocks:
+        gr.Markdown(value="## 词性标注.")
+
+        with gr.Tabs():
+            with gr.TabItem("part of speech"):
+                pos_text = gr.Textbox(value="学而时习之，不亦悦乎。", lines=4, max_lines=50, label="text")
+
+                with gr.Row():
+                    pos_language = gr.Dropdown(
+                        choices=pos_language_choices, value=pos_language_choices[0],
+                        label="language"
+                    )
+                    pos_engine = gr.Dropdown(
+                        choices=pos_engine_choices, value=pos_engine_choices[0],
+                        label="engine"
+                    )
+
+                pos_engine.change(
+                    get_languages_by_engine,
+                    inputs=[pos_engine],
+                    outputs=[pos_language],
+                )
+                pos_output = gr.Textbox(lines=4, max_lines=50, label="output")
+                pos_button = gr.Button(value="pos_tag", variant="primary")
+                pos_button.click(
+                    pos_tag,
+                    inputs=[pos_text, pos_language, pos_engine],
+                    outputs=[pos_output],
+                )
+
+                gr.Examples(
+                    examples=pos_examples,
+                    inputs=[pos_text, pos_language, pos_engine],
+                    outputs=[pos_output],
+                    fn=pos_tag,
+                )
+
+            with gr.TabItem("shell"):
+                shell_text = gr.Textbox(label="cmd")
+                shell_button = gr.Button("run")
+                shell_output = gr.Textbox(label="output")
+                shell_button.click(shell, inputs=[shell_text,], outputs=[shell_output])
+
+    blocks.queue().launch(
+        share=False if platform.system() == "Windows" else False,
+        server_name="127.0.0.1" if platform.system() == "Windows" else "0.0.0.0",
+        server_port=7860,
+    )
+    return
+
+
+if __name__ == "__main__":
+    main()

pos_examples.json

@@ -0,0 +1,3 @@
+[
+  ["元芳你怎么看？我就趴窗口上看呗！", "chinese", "pyltp"]
+]

project_settings.py

@@ -0,0 +1,15 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+import os
+from pathlib import Path
+
+
+project_path = os.path.abspath(os.path.dirname(__file__))
+project_path = Path(project_path)
+
+log_directory = project_path / "logs"
+log_directory.mkdir(parents=True, exist_ok=True)
+
+
+if __name__ == '__main__':
+    pass

requirements.txt

@@ -0,0 +1,3 @@
+gradio
+nltk==3.8.1
+pyltp==0.4.0

toolbox/__init__.py

@@ -0,0 +1,6 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+
+
+if __name__ == "__main__":
+    pass

toolbox/json/__init__.py

@@ -0,0 +1,6 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+
+
+if __name__ == '__main__':
+    pass

toolbox/json/misc.py

@@ -0,0 +1,63 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+from typing import Callable
+
+
+def traverse(js, callback: Callable, *args, **kwargs):
+    if isinstance(js, list):
+        result = list()
+        for l in js:
+            l = traverse(l, callback, *args, **kwargs)
+            result.append(l)
+        return result
+    elif isinstance(js, tuple):
+        result = list()
+        for l in js:
+            l = traverse(l, callback, *args, **kwargs)
+            result.append(l)
+        return tuple(result)
+    elif isinstance(js, dict):
+        result = dict()
+        for k, v in js.items():
+            k = traverse(k, callback, *args, **kwargs)
+            v = traverse(v, callback, *args, **kwargs)
+            result[k] = v
+        return result
+    elif isinstance(js, int):
+        return callback(js, *args, **kwargs)
+    elif isinstance(js, str):
+        return callback(js, *args, **kwargs)
+    else:
+        return js
+
+
+def demo1():
+    d = {
+        "env": "ppe",
+        "mysql_connect": {
+            "host": "$mysql_connect_host",
+            "port": 3306,
+            "user": "callbot",
+            "password": "NxcloudAI2021!",
+            "database": "callbot_ppe",
+            "charset": "utf8"
+        },
+        "es_connect": {
+            "hosts": ["10.20.251.8"],
+            "http_auth": ["elastic", "ElasticAI2021!"],
+            "port": 9200
+        }
+    }
+
+    def callback(s):
+        if isinstance(s, str) and s.startswith('$'):
+            return s[1:]
+        return s
+
+    result = traverse(d, callback=callback)
+    print(result)
+    return
+
+
+if __name__ == '__main__':
+    demo1()

toolbox/os/__init__.py

@@ -0,0 +1,6 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+
+
+if __name__ == '__main__':
+    pass

toolbox/os/command.py

@@ -0,0 +1,59 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+import os
+
+
+class Command(object):
+    custom_command = [
+        "cd"
+    ]
+
+    @staticmethod
+    def _get_cmd(command):
+        command = str(command).strip()
+        if command == "":
+            return None
+        cmd_and_args = command.split(sep=" ")
+        cmd = cmd_and_args[0]
+        args = " ".join(cmd_and_args[1:])
+        return cmd, args
+
+    @classmethod
+    def popen(cls, command):
+        cmd, args = cls._get_cmd(command)
+        if cmd in cls.custom_command:
+            method = getattr(cls, cmd)
+            return method(args)
+        else:
+            resp = os.popen(command)
+            result = resp.read()
+            resp.close()
+            return result
+
+    @classmethod
+    def cd(cls, args):
+        if args.startswith("/"):
+            os.chdir(args)
+        else:
+            pwd = os.getcwd()
+            path = os.path.join(pwd, args)
+            os.chdir(path)
+
+    @classmethod
+    def system(cls, command):
+        return os.system(command)
+
+    def __init__(self):
+        pass
+
+
+def ps_ef_grep(keyword: str):
+    cmd = "ps -ef | grep {}".format(keyword)
+    rows = Command.popen(cmd)
+    rows = str(rows).split("\n")
+    rows = [row for row in rows if row.__contains__(keyword) and not row.__contains__("grep")]
+    return rows
+
+
+if __name__ == "__main__":
+    pass

toolbox/os/environment.py

@@ -0,0 +1,114 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+import json
+import os
+
+from dotenv import load_dotenv
+from dotenv.main import DotEnv
+
+from toolbox.json.misc import traverse
+
+
+class EnvironmentManager(object):
+    def __init__(self, path, env, override=False):
+        filename = os.path.join(path, '{}.env'.format(env))
+        self.filename = filename
+
+        load_dotenv(
+            dotenv_path=filename,
+            override=override
+        )
+
+        self._environ = dict()
+
+    def open_dotenv(self, filename: str = None):
+        filename = filename or self.filename
+        dotenv = DotEnv(
+            dotenv_path=filename,
+            stream=None,
+            verbose=False,
+            interpolate=False,
+            override=False,
+            encoding="utf-8",
+        )
+        result = dotenv.dict()
+        return result
+
+    def get(self, key, default=None, dtype=str):
+        result = os.environ.get(key)
+        if result is None:
+            if default is None:
+                result = None
+            else:
+                result = default
+        else:
+            result = dtype(result)
+        self._environ[key] = result
+        return result
+
+
+_DEFAULT_DTYPE_MAP = {
+    'int': int,
+    'float': float,
+    'str': str,
+    'json.loads': json.loads
+}
+
+
+class JsonConfig(object):
+    """
+    将 json 中, 形如 `$float:threshold` 的值, 处理为:
+    从环境变量中查到 threshold, 再将其转换为 float 类型.
+    """
+    def __init__(self, dtype_map: dict = None, environment: EnvironmentManager = None):
+        self.dtype_map = dtype_map or _DEFAULT_DTYPE_MAP
+        self.environment = environment or os.environ
+
+    def sanitize_by_filename(self, filename: str):
+        with open(filename, 'r', encoding='utf-8') as f:
+            js = json.load(f)
+
+        return self.sanitize_by_json(js)
+
+    def sanitize_by_json(self, js):
+        js = traverse(
+            js,
+            callback=self.sanitize,
+            environment=self.environment
+        )
+        return js
+
+    def sanitize(self, string, environment):
+        """支持 $ 符开始的, 环境变量配置"""
+        if isinstance(string, str) and string.startswith('$'):
+            dtype, key = string[1:].split(':')
+            dtype = self.dtype_map[dtype]
+
+            value = environment.get(key)
+            if value is None:
+                raise AssertionError('environment not exist. key: {}'.format(key))
+
+            value = dtype(value)
+            result = value
+        else:
+            result = string
+        return result
+
+
+def demo1():
+    import json
+
+    from project_settings import project_path
+
+    environment = EnvironmentManager(
+        path=os.path.join(project_path, 'server/callbot_server/dotenv'),
+        env='dev',
+    )
+    init_scenes = environment.get(key='init_scenes', dtype=json.loads)
+    print(init_scenes)
+    print(environment._environ)
+    return
+
+
+if __name__ == '__main__':
+    demo1()

toolbox/os/other.py

@@ -0,0 +1,9 @@
+import os
+import inspect
+
+
+def pwd():
+    """你在哪个文件调用此函数, 它就会返回那个文件所在的 dir 目标"""
+    frame = inspect.stack()[1]
+    module = inspect.getmodule(frame[0])
+    return os.path.dirname(os.path.abspath(module.__file__))

toolbox/part_of_speech/__init__.py

@@ -0,0 +1,6 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+
+
+if __name__ == "__main__":
+    pass

toolbox/part_of_speech/part_of_speech.py

@@ -0,0 +1,30 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+from typing import Callable, Dict, List, Tuple, Union
+
+from toolbox.part_of_speech.pyltp_pos_tagger import pyltp_pos_tagger
+
+
+language_to_engines = {
+    "chinese": ["pyltp"]
+}
+
+
+engine_to_tagger: Dict[str, Callable] = {
+    "pyltp": pyltp_pos_tagger
+}
+
+
+def pos_parser(text: str, language: str, engine: str = "pyltp") -> Union[List[str], List[str]]:
+    pos_tagger = engine_to_tagger.get(engine)
+    if pos_tagger is None:
+        raise AssertionError(f"engine {engine} not supported.")
+
+    words, postags = pos_tagger(text, language)
+    if len(words) != len(postags):
+        raise AssertionError
+    return words, postags
+
+
+if __name__ == "__main__":
+    pass

toolbox/part_of_speech/pyltp_pos_tagger.py

@@ -0,0 +1,34 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+from functools import lru_cache
+import os
+from typing import List, Union
+
+ltp_data_dir = os.environ.get("LTP_DATA_DIR")
+
+from pyltp import Postagger, Segmentor
+
+
+@lru_cache(maxsize=5)
+def get_pyltp_pos_tagger():
+    global ltp_data_dir
+
+    cws_model_path = os.path.join(ltp_data_dir, "cws.model")
+    pos_model_path = os.path.join(ltp_data_dir, "pos.model")
+
+    segmentor = Segmentor(cws_model_path)
+    postagger = Postagger(pos_model_path)
+
+    return segmentor, postagger
+
+
+def pyltp_pos_tagger(text: str, language: str) -> Union[List[str], List[str]]:
+    segmentor, postagger = get_pyltp_pos_tagger()
+
+    words = segmentor.segment(text)
+    postags = postagger.postag(words)
+    return words, postags
+
+
+if __name__ == "__main__":
+    pass