Upload 4 files

README.md

@@ -1,3 +1,66 @@
----
-license: mit
----
+#### 1D-CNN-MC-toxicity-classifier-ru
+(One-Dimensional Convolutional Neural Network with Multi-Channel input)
+
+Architectural visualization:
+
+![](https://i.imgur.com/skbLM6w.png)
+
+Total parameters: 503249
+
+##### Test Accuracy: 94.44%
+##### Training Accuracy: 97.46%
+
+This model is developed for binary classification of Cyrillic text.
+
+##### A dataset of 75093 negative rows and 75093 positive rows was used for training.
+
+##### Recommended length of the input sequence: 25 - 400 Cyrillic characters.
+
+##### Simplifications of the dataset strings: 
+Removing extra spaces.
+
+Replacing capital letters with small letters. (Я -> я).
+
+Removing any non-Cyrillic characters, including prefixes. (Remove: z, !, ., #, 4, &... etc)
+
+Replacing ё with e.
+
+### Example of use:
+
+    import numpy as np
+    from tensorflow import keras
+    from tensorflow.keras.preprocessing.text import tokenizer_from_json
+    from safetensors.numpy import load_file
+    from tensorflow.keras.preprocessing.sequence import pad_sequences
+    import os
+    import re
+    # Название папки, где хранится модель
+    model_dir = 'model'
+    max_len = 400
+    # Загрузка архитектуры модели
+    with open(os.path.join(model_dir, 'model_architecture.json'), 'r', encoding='utf-8') as json_file:
+        model_json = json_file.read()
+    model = keras.models.model_from_json(model_json)
+    # Загрузка весов из safetensors
+    state_dict = load_file(os.path.join(model_dir, 'model_weights.safetensors'))
+    weights = [state_dict[f'weight_{i}'] for i in range(len(state_dict))]
+    model.set_weights(weights)
+    # Загрузка токенизатора
+    with open(os.path.join(model_dir, 'tokenizer.json'), 'r', encoding='utf-8') as f:
+        tokenizer_json = f.read()
+    tokenizer = tokenizer_from_json(tokenizer_json)
+    def predict_toxicity(text):
+        sequences = tokenizer.texts_to_sequences([text])
+        padded = pad_sequences(sequences, maxlen=max_len, padding='post', truncating='post')
+        probability = model.predict(padded)[0][0]
+        class_label = "toxic" if probability >= 0.5 else "normal"
+        return class_label, probability
+    # Пример использования
+    text = "Да какой идиот сделал эту НС?"
+    class_label, probability = predict_toxicity(text)
+    print(f"Text: {text}")
+    print(f"Class: {class_label} ({probability:.2%})")
+
+###### Output: 
+Text: Да какой идиот сделал эту НС?
+Class: toxic (99.35%)

model_architecture.json

@@ -0,0 +1 @@
+{"module": "keras", "class_name": "Sequential", "config": {"name": "sequential", "trainable": true, "dtype": {"module": "keras", "class_name": "DTypePolicy", "config": {"name": "float32"}, "registered_name": null}, "layers": [{"module": "keras.layers", "class_name": "InputLayer", "config": {"batch_shape": [8, 400], "dtype": "float32", "sparse": false, "name": "input_layer"}, "registered_name": null}, {"module": "keras.layers", "class_name": "Embedding", "config": {"name": "embedding", "trainable": true, "dtype": {"module": "keras", "class_name": "DTypePolicy", "config": {"name": "float32"}, "registered_name": null}, "input_dim": 10002, "output_dim": 48, "embeddings_initializer": {"module": "keras.initializers", "class_name": "RandomUniform", "config": {"seed": null, "minval": -0.05, "maxval": 0.05}, "registered_name": null}, "embeddings_regularizer": null, "activity_regularizer": null, "embeddings_constraint": null, "mask_zero": false}, "registered_name": null, "build_config": {"input_shape": [8, 400]}}, {"module": "keras.layers", "class_name": "Conv1D", "config": {"name": "conv1d", "trainable": true, "dtype": {"module": "keras", "class_name": "DTypePolicy", "config": {"name": "float32"}, "registered_name": null}, "filters": 48, "kernel_size": [3], "strides": [1], "padding": "valid", "data_format": "channels_last", "dilation_rate": [1], "groups": 1, "activation": "relu", "use_bias": true, "kernel_initializer": {"module": "keras.initializers", "class_name": "GlorotUniform", "config": {"seed": null}, "registered_name": null}, "bias_initializer": {"module": "keras.initializers", "class_name": "Zeros", "config": {}, "registered_name": null}, "kernel_regularizer": null, "bias_regularizer": null, "activity_regularizer": null, "kernel_constraint": null, "bias_constraint": null}, "registered_name": null, "build_config": {"input_shape": [8, 400, 48]}}, {"module": "keras.layers", "class_name": "MaxPooling1D", "config": {"name": "max_pooling1d", "trainable": true, "dtype": {"module": "keras", "class_name": "DTypePolicy", "config": {"name": "float32"}, "registered_name": null}, "pool_size": [2], "padding": "valid", "strides": [2], "data_format": "channels_last"}, "registered_name": null}, {"module": "keras.layers", "class_name": "Conv1D", "config": {"name": "conv1d_1", "trainable": true, "dtype": {"module": "keras", "class_name": "DTypePolicy", "config": {"name": "float32"}, "registered_name": null}, "filters": 16, "kernel_size": [5], "strides": [1], "padding": "valid", "data_format": "channels_last", "dilation_rate": [1], "groups": 1, "activation": "relu", "use_bias": true, "kernel_initializer": {"module": "keras.initializers", "class_name": "GlorotUniform", "config": {"seed": null}, "registered_name": null}, "bias_initializer": {"module": "keras.initializers", "class_name": "Zeros", "config": {}, "registered_name": null}, "kernel_regularizer": null, "bias_regularizer": null, "activity_regularizer": null, "kernel_constraint": null, "bias_constraint": null}, "registered_name": null, "build_config": {"input_shape": [8, 199, 48]}}, {"module": "keras.layers", "class_name": "MaxPooling1D", "config": {"name": "max_pooling1d_1", "trainable": true, "dtype": {"module": "keras", "class_name": "DTypePolicy", "config": {"name": "float32"}, "registered_name": null}, "pool_size": [2], "padding": "valid", "strides": [2], "data_format": "channels_last"}, "registered_name": null}, {"module": "keras.layers", "class_name": "Flatten", "config": {"name": "flatten", "trainable": true, "dtype": {"module": "keras", "class_name": "DTypePolicy", "config": {"name": "float32"}, "registered_name": null}, "data_format": "channels_last"}, "registered_name": null, "build_config": {"input_shape": [8, 97, 16]}}, {"module": "keras.layers", "class_name": "Dense", "config": {"name": "dense", "trainable": true, "dtype": {"module": "keras", "class_name": "DTypePolicy", "config": {"name": "float32"}, "registered_name": null}, "units": 8, "activation": "relu", "use_bias": true, "kernel_initializer": {"module": "keras.initializers", "class_name": "GlorotUniform", "config": {"seed": null}, "registered_name": null}, "bias_initializer": {"module": "keras.initializers", "class_name": "Zeros", "config": {}, "registered_name": null}, "kernel_regularizer": null, "bias_regularizer": null, "kernel_constraint": null, "bias_constraint": null}, "registered_name": null, "build_config": {"input_shape": [8, 1552]}}, {"module": "keras.layers", "class_name": "Dense", "config": {"name": "dense_1", "trainable": true, "dtype": {"module": "keras", "class_name": "DTypePolicy", "config": {"name": "float32"}, "registered_name": null}, "units": 1, "activation": "sigmoid", "use_bias": true, "kernel_initializer": {"module": "keras.initializers", "class_name": "GlorotUniform", "config": {"seed": null}, "registered_name": null}, "bias_initializer": {"module": "keras.initializers", "class_name": "Zeros", "config": {}, "registered_name": null}, "kernel_regularizer": null, "bias_regularizer": null, "kernel_constraint": null, "bias_constraint": null}, "registered_name": null, "build_config": {"input_shape": [8, 8]}}], "build_input_shape": [8, 400]}, "registered_name": null, "build_config": {"input_shape": [8, 400]}, "compile_config": {"optimizer": {"module": "keras.optimizers", "class_name": "Adam", "config": {"name": "adam", "learning_rate": 0.0010000000474974513, "weight_decay": null, "clipnorm": null, "global_clipnorm": null, "clipvalue": null, "use_ema": false, "ema_momentum": 0.99, "ema_overwrite_frequency": null, "loss_scale_factor": null, "gradient_accumulation_steps": null, "beta_1": 0.9, "beta_2": 0.999, "epsilon": 1e-07, "amsgrad": false}, "registered_name": null}, "loss": "binary_crossentropy", "loss_weights": null, "metrics": ["accuracy"], "weighted_metrics": null, "run_eagerly": false, "steps_per_execution": 1, "jit_compile": false}}

model_weights.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8349a9b2f80b5c1a42053aea4a3959eb235d6c5845a585a0b38ccb1953f89f8c
+size 2014060