Replace model-s-cpu with load_model_s (model-s-512 model-s-512-v2 on hf)

.gitignore

@@ -143,3 +143,5 @@ links/
 node_modules
 install-sw.sh
 install-sw1.sh
+win10-install-memo.txt
+model-s

st_mlbee/__init__.py

@@ -1,5 +1,5 @@
 """Init."""
 __version__ = "0.1.0a2"
-from .st_mlbee import st_mlbee
+from st_mlbee.st_mlbee import st_mlbee
 
 __all__ = ("st_mlbee",)

st_mlbee/gen_cmat.py

@@ -9,14 +9,27 @@ import numpy as np
 from tqdm import tqdm
 
 # from model_pool import load_model_s
-from hf_model_s_cpu import model_s
-from logzero import logger
+# from hf_model_s_cpu import model_s  # load_model_s directly
+from st_mlbee.load_model_s import load_model_s
 
-from st_mlbee.cos_matrix2 import cos_matrix2
+# from logzero import logger
+from loguru import logger
+
+# from st_mlbee.cos_matrix2 import cos_matrix2
+from .cos_matrix2 import cos_matrix2
+
+_ = """
+try:
+    model_s = load_model_s()
+except Exception as exc:
+    logger.erorr(exc)
+    raise
+"""
 
 try:
     # model = model_s()
-    model = model_s(alive_bar_on=True)
+    # model = model_s(alive_bar_on=True)
+    model = load_model_s()
 except Exception as _:
     logger.error(_)
     raise

st_mlbee/load_model_s.py

@@ -0,0 +1,27 @@
+r"""
+Load model_s from hf.
+
+cf aslo align-model-pool\model_pool\load_model.py and ycco make-upload-model-s.ipynb.
+"""
+import torch
+import joblib
+from huggingface_hub import hf_hub_download
+from loguru import logger
+
+try:
+    loc = hf_hub_download("mikeee/model_s_512", "model-s", local_dir=".")
+except Exception as exc:
+    logger.error(exc)
+    raise SystemExit(1) from exc
+
+
+def load_model_s(model_file=None):
+    """Load a model from hf."""
+    if model_file is None:
+        model_file = loc
+    try:
+        model = joblib.load(model_file)
+    except Exception as exc:
+        logger.error(exc)
+        raise
+    return model

tests/test_gen_cmat_old.py

@@ -0,0 +1,19 @@
+"""Test old gen_cmat."""
+from loadtext import loadtext
+from st_mlbee.gen_cmat import gen_cmat
+
+
+def test_gen_cmat_old():
+    """Test old gen_cam."""
+    paras1 = loadtext("data/sternstunden04-en.txt", splitlines=True)
+    paras2 = loadtext("data/sternstunden04-de.txt", True)
+
+    cmat = gen_cmat(paras1, paras2)
+    len1, len2 = len(paras1), len(paras2)
+
+    # note the order
+    assert cmat.shape == (len2, len1)
+
+
+if __name__ == "__main__":
+    test_gen_cmat_old()

tests1/__init__.py

@@ -0,0 +1 @@
+"""Init."""

tests1/test_gen_cmat_old.py

@@ -0,0 +1,19 @@
+"""Test old gen_cmat."""
+from loadtext import loadtext
+from st_mlbee.gen_cmat import gen_cmat
+
+
+def test_gen_cmat_old():
+    """Test old gen_cam."""
+    paras1 = loadtext("data/sternstunden04-en.txt", splitlines=True)
+    paras2 = loadtext("data/sternstunden04-de.txt", True)
+
+    cmat = gen_cmat(paras1, paras2)
+    len1, len2 = len(paras1), len(paras2)
+
+    # note the order
+    assert cmat.shape == (len2, len1)
+
+
+if __name__ == "__main__":
+    test_gen_cmat_old()