{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from datasets import load_dataset\n",
    "import pandas as pd \n",
    "from pymongo import MongoClient\n",
    "from transformers import BertTokenizer, BertForMaskedLM, DPRContextEncoderTokenizer,DPRContextEncoder;\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "import numpy as np\n",
    "import re\n",
    "import pandas as pd\n",
    "from nltk.stem import WordNetLemmatizer\n",
    "from nltk.corpus import stopwords as nltk_stopwords\n",
    "from transformers import BertTokenizer, BertModel, AutoTokenizer\n",
    "from sklearn.metrics.pairwise import cosine_similarity\n",
    "import torch\n",
    "from pymongo import MongoClient\n",
    "import torch.nn.functional as F"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024-08-27 17:58:58 INFO: Checking for updates to resources.json in case models have been updated.  Note: this behavior can be turned off with download_method=None or download_method=DownloadMethod.REUSE_RESOURCES\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8ff25d869f4a47a8b1645d6a09afdb49",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading https://raw.githubusercontent.com/stanfordnlp/stanza-resources/main/resources_1.8.0.json:   0%|   …"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024-08-27 17:58:58 INFO: Downloaded file to C:\\Users\\info\\stanza_resources\\resources.json\n",
      "2024-08-27 17:58:59 INFO: Loading these models for language: tr (Turkish):\n",
      "=============================\n",
      "| Processor | Package       |\n",
      "-----------------------------\n",
      "| tokenize  | imst          |\n",
      "| mwt       | imst          |\n",
      "| pos       | imst_charlm   |\n",
      "| lemma     | imst_nocharlm |\n",
      "| depparse  | imst_charlm   |\n",
      "| ner       | starlang      |\n",
      "=============================\n",
      "\n",
      "2024-08-27 17:58:59 INFO: Using device: cpu\n",
      "2024-08-27 17:58:59 INFO: Loading: tokenize\n",
      "c:\\gitProjects\\yeni\\.venv\\lib\\site-packages\\stanza\\models\\tokenization\\trainer.py:82: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.\n",
      "  checkpoint = torch.load(filename, lambda storage, loc: storage)\n",
      "2024-08-27 17:58:59 INFO: Loading: mwt\n",
      "c:\\gitProjects\\yeni\\.venv\\lib\\site-packages\\stanza\\models\\mwt\\trainer.py:170: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.\n",
      "  checkpoint = torch.load(filename, lambda storage, loc: storage)\n",
      "2024-08-27 17:58:59 INFO: Loading: pos\n",
      "c:\\gitProjects\\yeni\\.venv\\lib\\site-packages\\stanza\\models\\pos\\trainer.py:139: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.\n",
      "  checkpoint = torch.load(filename, lambda storage, loc: storage)\n",
      "c:\\gitProjects\\yeni\\.venv\\lib\\site-packages\\stanza\\models\\common\\pretrain.py:56: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.\n",
      "  data = torch.load(self.filename, lambda storage, loc: storage)\n",
      "c:\\gitProjects\\yeni\\.venv\\lib\\site-packages\\stanza\\models\\common\\char_model.py:271: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.\n",
      "  state = torch.load(filename, lambda storage, loc: storage)\n",
      "2024-08-27 17:58:59 INFO: Loading: lemma\n",
      "c:\\gitProjects\\yeni\\.venv\\lib\\site-packages\\stanza\\models\\lemma\\trainer.py:236: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.\n",
      "  checkpoint = torch.load(filename, lambda storage, loc: storage)\n",
      "2024-08-27 17:58:59 INFO: Loading: depparse\n",
      "c:\\gitProjects\\yeni\\.venv\\lib\\site-packages\\stanza\\models\\depparse\\trainer.py:194: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.\n",
      "  checkpoint = torch.load(filename, lambda storage, loc: storage)\n",
      "2024-08-27 17:58:59 INFO: Loading: ner\n",
      "c:\\gitProjects\\yeni\\.venv\\lib\\site-packages\\stanza\\models\\ner\\trainer.py:197: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.\n",
      "  checkpoint = torch.load(filename, lambda storage, loc: storage)\n",
      "2024-08-27 17:59:00 INFO: Done loading processors!\n"
     ]
    },
    {
     "ename": "TypeError",
     "evalue": "'Document' object is not iterable",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "Cell \u001b[1;32mIn[18], line 94\u001b[0m\n\u001b[0;32m     92\u001b[0m \u001b[38;5;66;03m# ---------------------------------Verilerin kaydedilmesi-------------------------------------\u001b[39;00m\n\u001b[0;32m     93\u001b[0m processor \u001b[38;5;241m=\u001b[39m DataProcessor(input_csv\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mtexts_egitim.csv\u001b[39m\u001b[38;5;124m\"\u001b[39m, output_csv\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcleaned_data4.csv\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m---> 94\u001b[0m \u001b[43mprocessor\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mmain_pipeline\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n",
      "Cell \u001b[1;32mIn[18], line 74\u001b[0m, in \u001b[0;36mDataProcessor.main_pipeline\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m     71\u001b[0m df[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mkısaltılmıs_metin\u001b[39m\u001b[38;5;124m'\u001b[39m] \u001b[38;5;241m=\u001b[39m df[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mmetinler\u001b[39m\u001b[38;5;124m'\u001b[39m]\u001b[38;5;241m.\u001b[39mapply(filter_text)\n\u001b[0;32m     73\u001b[0m \u001b[38;5;66;03m# Metinleri kısalt\u001b[39;00m\n\u001b[1;32m---> 74\u001b[0m df[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mkısaltılmıs_metin\u001b[39m\u001b[38;5;124m'\u001b[39m] \u001b[38;5;241m=\u001b[39m \u001b[43mdf\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mmetinler\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m]\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mapply\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43;01mlambda\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43mx\u001b[49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43mtruncate_text_meaningful\u001b[49m\u001b[43m(\u001b[49m\u001b[43mx\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmax_len\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mmax_words\u001b[49m\u001b[43m)\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m     76\u001b[0m \u001b[38;5;66;03m# Tokenize et ve padding uygula\u001b[39;00m\n\u001b[0;32m     77\u001b[0m padded_tokens \u001b[38;5;241m=\u001b[39m tokenize_and_pad(df[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mkısaltılmıs_metin\u001b[39m\u001b[38;5;124m'\u001b[39m]\u001b[38;5;241m.\u001b[39mtolist(), model_name\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mmodel_name)\n",
      "File \u001b[1;32mc:\\gitProjects\\yeni\\.venv\\lib\\site-packages\\pandas\\core\\series.py:4924\u001b[0m, in \u001b[0;36mSeries.apply\u001b[1;34m(self, func, convert_dtype, args, by_row, **kwargs)\u001b[0m\n\u001b[0;32m   4789\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mapply\u001b[39m(\n\u001b[0;32m   4790\u001b[0m     \u001b[38;5;28mself\u001b[39m,\n\u001b[0;32m   4791\u001b[0m     func: AggFuncType,\n\u001b[1;32m   (...)\u001b[0m\n\u001b[0;32m   4796\u001b[0m     \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs,\n\u001b[0;32m   4797\u001b[0m ) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m DataFrame \u001b[38;5;241m|\u001b[39m Series:\n\u001b[0;32m   4798\u001b[0m \u001b[38;5;250m    \u001b[39m\u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[0;32m   4799\u001b[0m \u001b[38;5;124;03m    Invoke function on values of Series.\u001b[39;00m\n\u001b[0;32m   4800\u001b[0m \n\u001b[1;32m   (...)\u001b[0m\n\u001b[0;32m   4915\u001b[0m \u001b[38;5;124;03m    dtype: float64\u001b[39;00m\n\u001b[0;32m   4916\u001b[0m \u001b[38;5;124;03m    \"\"\"\u001b[39;00m\n\u001b[0;32m   4917\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mSeriesApply\u001b[49m\u001b[43m(\u001b[49m\n\u001b[0;32m   4918\u001b[0m \u001b[43m        \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[0;32m   4919\u001b[0m \u001b[43m        \u001b[49m\u001b[43mfunc\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m   4920\u001b[0m \u001b[43m        \u001b[49m\u001b[43mconvert_dtype\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mconvert_dtype\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m   4921\u001b[0m \u001b[43m        \u001b[49m\u001b[43mby_row\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mby_row\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m   4922\u001b[0m \u001b[43m        \u001b[49m\u001b[43margs\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m   4923\u001b[0m \u001b[43m        \u001b[49m\u001b[43mkwargs\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m-> 4924\u001b[0m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mapply\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[1;32mc:\\gitProjects\\yeni\\.venv\\lib\\site-packages\\pandas\\core\\apply.py:1427\u001b[0m, in \u001b[0;36mSeriesApply.apply\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m   1424\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mapply_compat()\n\u001b[0;32m   1426\u001b[0m \u001b[38;5;66;03m# self.func is Callable\u001b[39;00m\n\u001b[1;32m-> 1427\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mapply_standard\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[1;32mc:\\gitProjects\\yeni\\.venv\\lib\\site-packages\\pandas\\core\\apply.py:1507\u001b[0m, in \u001b[0;36mSeriesApply.apply_standard\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m   1501\u001b[0m \u001b[38;5;66;03m# row-wise access\u001b[39;00m\n\u001b[0;32m   1502\u001b[0m \u001b[38;5;66;03m# apply doesn't have a `na_action` keyword and for backward compat reasons\u001b[39;00m\n\u001b[0;32m   1503\u001b[0m \u001b[38;5;66;03m# we need to give `na_action=\"ignore\"` for categorical data.\u001b[39;00m\n\u001b[0;32m   1504\u001b[0m \u001b[38;5;66;03m# TODO: remove the `na_action=\"ignore\"` when that default has been changed in\u001b[39;00m\n\u001b[0;32m   1505\u001b[0m \u001b[38;5;66;03m#  Categorical (GH51645).\u001b[39;00m\n\u001b[0;32m   1506\u001b[0m action \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mignore\u001b[39m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(obj\u001b[38;5;241m.\u001b[39mdtype, CategoricalDtype) \u001b[38;5;28;01melse\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[1;32m-> 1507\u001b[0m mapped \u001b[38;5;241m=\u001b[39m \u001b[43mobj\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_map_values\u001b[49m\u001b[43m(\u001b[49m\n\u001b[0;32m   1508\u001b[0m \u001b[43m    \u001b[49m\u001b[43mmapper\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcurried\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mna_action\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43maction\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mconvert\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mconvert_dtype\u001b[49m\n\u001b[0;32m   1509\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m   1511\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mlen\u001b[39m(mapped) \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(mapped[\u001b[38;5;241m0\u001b[39m], ABCSeries):\n\u001b[0;32m   1512\u001b[0m     \u001b[38;5;66;03m# GH#43986 Need to do list(mapped) in order to get treated as nested\u001b[39;00m\n\u001b[0;32m   1513\u001b[0m     \u001b[38;5;66;03m#  See also GH#25959 regarding EA support\u001b[39;00m\n\u001b[0;32m   1514\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m obj\u001b[38;5;241m.\u001b[39m_constructor_expanddim(\u001b[38;5;28mlist\u001b[39m(mapped), index\u001b[38;5;241m=\u001b[39mobj\u001b[38;5;241m.\u001b[39mindex)\n",
      "File \u001b[1;32mc:\\gitProjects\\yeni\\.venv\\lib\\site-packages\\pandas\\core\\base.py:921\u001b[0m, in \u001b[0;36mIndexOpsMixin._map_values\u001b[1;34m(self, mapper, na_action, convert)\u001b[0m\n\u001b[0;32m    918\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(arr, ExtensionArray):\n\u001b[0;32m    919\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m arr\u001b[38;5;241m.\u001b[39mmap(mapper, na_action\u001b[38;5;241m=\u001b[39mna_action)\n\u001b[1;32m--> 921\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43malgorithms\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mmap_array\u001b[49m\u001b[43m(\u001b[49m\u001b[43marr\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmapper\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mna_action\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mna_action\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mconvert\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mconvert\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[1;32mc:\\gitProjects\\yeni\\.venv\\lib\\site-packages\\pandas\\core\\algorithms.py:1743\u001b[0m, in \u001b[0;36mmap_array\u001b[1;34m(arr, mapper, na_action, convert)\u001b[0m\n\u001b[0;32m   1741\u001b[0m values \u001b[38;5;241m=\u001b[39m arr\u001b[38;5;241m.\u001b[39mastype(\u001b[38;5;28mobject\u001b[39m, copy\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mFalse\u001b[39;00m)\n\u001b[0;32m   1742\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m na_action \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m-> 1743\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mlib\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mmap_infer\u001b[49m\u001b[43m(\u001b[49m\u001b[43mvalues\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmapper\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mconvert\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mconvert\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m   1744\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m   1745\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m lib\u001b[38;5;241m.\u001b[39mmap_infer_mask(\n\u001b[0;32m   1746\u001b[0m         values, mapper, mask\u001b[38;5;241m=\u001b[39misna(values)\u001b[38;5;241m.\u001b[39mview(np\u001b[38;5;241m.\u001b[39muint8), convert\u001b[38;5;241m=\u001b[39mconvert\n\u001b[0;32m   1747\u001b[0m     )\n",
      "File \u001b[1;32mlib.pyx:2972\u001b[0m, in \u001b[0;36mpandas._libs.lib.map_infer\u001b[1;34m()\u001b[0m\n",
      "Cell \u001b[1;32mIn[18], line 74\u001b[0m, in \u001b[0;36mDataProcessor.main_pipeline.<locals>.<lambda>\u001b[1;34m(x)\u001b[0m\n\u001b[0;32m     71\u001b[0m df[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mkısaltılmıs_metin\u001b[39m\u001b[38;5;124m'\u001b[39m] \u001b[38;5;241m=\u001b[39m df[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mmetinler\u001b[39m\u001b[38;5;124m'\u001b[39m]\u001b[38;5;241m.\u001b[39mapply(filter_text)\n\u001b[0;32m     73\u001b[0m \u001b[38;5;66;03m# Metinleri kısalt\u001b[39;00m\n\u001b[1;32m---> 74\u001b[0m df[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mkısaltılmıs_metin\u001b[39m\u001b[38;5;124m'\u001b[39m] \u001b[38;5;241m=\u001b[39m df[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mmetinler\u001b[39m\u001b[38;5;124m'\u001b[39m]\u001b[38;5;241m.\u001b[39mapply(\u001b[38;5;28;01mlambda\u001b[39;00m x: \u001b[43mtruncate_text_meaningful\u001b[49m\u001b[43m(\u001b[49m\u001b[43mx\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmax_len\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mmax_words\u001b[49m\u001b[43m)\u001b[49m)\n\u001b[0;32m     76\u001b[0m \u001b[38;5;66;03m# Tokenize et ve padding uygula\u001b[39;00m\n\u001b[0;32m     77\u001b[0m padded_tokens \u001b[38;5;241m=\u001b[39m tokenize_and_pad(df[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mkısaltılmıs_metin\u001b[39m\u001b[38;5;124m'\u001b[39m]\u001b[38;5;241m.\u001b[39mtolist(), model_name\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mmodel_name)\n",
      "Cell \u001b[1;32mIn[18], line 31\u001b[0m, in \u001b[0;36mtruncate_text_meaningful\u001b[1;34m(text, max_len)\u001b[0m\n\u001b[0;32m     28\u001b[0m doc \u001b[38;5;241m=\u001b[39m nlp(text)\n\u001b[0;32m     30\u001b[0m \u001b[38;5;66;03m# Stop kelimeleri ve noktalama işaretlerini kaldır\u001b[39;00m\n\u001b[1;32m---> 31\u001b[0m tokens \u001b[38;5;241m=\u001b[39m [token\u001b[38;5;241m.\u001b[39mlemma_ \u001b[38;5;28;01mfor\u001b[39;00m token \u001b[38;5;129;01min\u001b[39;00m doc \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m token\u001b[38;5;241m.\u001b[39mis_stop \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m token\u001b[38;5;241m.\u001b[39mis_punct]\n\u001b[0;32m     33\u001b[0m \u001b[38;5;66;03m# Belirli bir uzunluktaki metni döndür\u001b[39;00m\n\u001b[0;32m     34\u001b[0m truncated_text \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124m \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;241m.\u001b[39mjoin(tokens[:max_len])\n",
      "\u001b[1;31mTypeError\u001b[0m: 'Document' object is not iterable"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import re\n",
    "from transformers import AutoTokenizer\n",
    "import spacy\n",
    "import stanza\n",
    "\n",
    "# ------------------------Cümlelerin boyutlarını ve stop words'leri tanımladığımız yer-----------------------------\n",
    "# Yüklediğiniz modele göre değiştirebilirsiniz\n",
    "nlp = stanza.Pipeline('tr')\n",
    "\n",
    "def preprocess_text(text):\n",
    "    doc = nlp(text)\n",
    "    tokens = [token.lemma_ for token in doc if not token.is_stop and not token.is_punct]\n",
    "    return \" \".join(tokens)\n",
    "\n",
    "def extract_keywords_and_subheadings(text):\n",
    "    doc = nlp(text)\n",
    "    keywords = []\n",
    "    subheadings = []\n",
    "    for ent in doc.ents:\n",
    "        if ent.label_ == \"ORG\" or ent.label_ == \"PERSON\":  # Örnek: Kurum veya kişi isimleri\n",
    "            keywords.append(ent.text)\n",
    "        elif ent.label_ == \"GPE\":  # Örnek: Yer isimleri\n",
    "            subheadings.append(ent.text)\n",
    "    return keywords, subheadings\n",
    "\n",
    "def truncate_text_meaningful(text, max_len=300):\n",
    "    # Önce metni tokenlere ayır\n",
    "    doc = nlp(text)\n",
    "\n",
    "    # Stop kelimeleri ve noktalama işaretlerini kaldır\n",
    "    tokens = [token.lemma_ for token in doc if not token.is_stop and not token.is_punct]\n",
    "\n",
    "    # Belirli bir uzunluktaki metni döndür\n",
    "    truncated_text = ' '.join(tokens[:max_len])\n",
    "\n",
    "    return truncated_text\n",
    "\n",
    "# ----------------------------------Tokenize etme fonksiyonu-----------------------------------\n",
    "def tokenize_and_pad(data, model_name='bert-base-uncased', max_length=512):\n",
    "    tokenizer = AutoTokenizer.from_pretrained(model_name)\n",
    "    encoded_input = tokenizer(data, padding=True, truncation=True, max_length=max_length)\n",
    "    return encoded_input\n",
    "\n",
    "class DataProcessor:\n",
    "    def __init__(self, input_csv, output_csv, max_words=300, model_name='dbmdz/distilbert-base-turkish-cased'):\n",
    "        self.input_csv = input_csv\n",
    "        self.output_csv = output_csv\n",
    "        self.max_words = max_words\n",
    "        self.model_name = model_name\n",
    "\n",
    "    def main_pipeline(self):\n",
    "        def filter_text(text):\n",
    "            # Dış bağlantılar ve kaynakçaları kaldır\n",
    "            text = re.sub(r'http\\S+|https\\S+|\\b(?:www\\.)?\\S+\\.\\w{2,4}\\b', '', text)\n",
    "            # Tarih ve sayıları kaldır\n",
    "            text = re.sub(r'\\d{4}-\\d{2}-\\d{2}|\\d{2}/\\d{2}/\\d{4}|\\d+', '', text)\n",
    "            # Sayıları kaldır\n",
    "            text = re.sub(r'\\d+', '', text)\n",
    "            # Kısa veya uzun kelimeleri kaldır\n",
    "            words = text.split()\n",
    "            words = [word for word in words if 2 <= len(word) <= 20]\n",
    "            return ' '.join(words)\n",
    "        \n",
    "        # UTF-8 encoding ile dosyayı okuyun\n",
    "        df = pd.read_csv(self.input_csv, encoding='utf-8')\n",
    "        \n",
    "        # Metinlerin sütun adını kontrol edin\n",
    "        if 'metinler' not in df.columns:\n",
    "            raise ValueError(\"CSV dosyasında 'metinler' adlı bir sütun bulunamadı. Lütfen sütun adını kontrol edin.\")\n",
    "        \n",
    "        df['kısaltılmıs_metin'] = df['metinler'].apply(filter_text)\n",
    "\n",
    "        # Metinleri kısalt\n",
    "        df['kısaltılmıs_metin'] = df['metinler'].apply(lambda x: truncate_text_meaningful(x, max_len=self.max_words))\n",
    "\n",
    "        # Tokenize et ve padding uygula\n",
    "        padded_tokens = tokenize_and_pad(df['kısaltılmıs_metin'].tolist(), model_name=self.model_name)\n",
    "        df['padded_tokens'] = padded_tokens['input_ids']\n",
    "\n",
    "        print(\"Kısaltılmış metinler:\")\n",
    "        print(df['kısaltılmıs_metin'].head())\n",
    "        print(\"Tokenize edilmiş ve padding uygulanmış veriler:\")\n",
    "        print(df[['kısaltılmıs_metin', 'padded_tokens']].head())\n",
    "\n",
    "        # Veriyi kaydet\n",
    "        self.save_cleaned_data(df)\n",
    "    \n",
    "    def save_cleaned_data(self, df):\n",
    "        df.to_csv(self.output_csv, index=False, encoding='utf-8')\n",
    "        print(f\"Temizlenmiş veri '{self.output_csv}' dosyasına kaydedildi.\")\n",
    "\n",
    "# ---------------------------------Verilerin kaydedilmesi-------------------------------------\n",
    "processor = DataProcessor(input_csv=\"texts_egitim.csv\", output_csv=\"cleaned_data4.csv\")\n",
    "processor.main_pipeline()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "ename": "ValueError",
     "evalue": "The language 'turkish' is not supported.",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "Cell \u001b[1;32mIn[4], line 6\u001b[0m\n\u001b[0;32m      3\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mnltk\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mstem\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01msnowball\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m SnowballStemmer\n\u001b[0;32m      5\u001b[0m \u001b[38;5;66;03m# Türkçe stemmer\u001b[39;00m\n\u001b[1;32m----> 6\u001b[0m stemmer \u001b[38;5;241m=\u001b[39m \u001b[43mSnowballStemmer\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mturkish\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m)\u001b[49m\n\u001b[0;32m      8\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mstem_text\u001b[39m(text):\n\u001b[0;32m      9\u001b[0m     words \u001b[38;5;241m=\u001b[39m text\u001b[38;5;241m.\u001b[39msplit()\n",
      "File \u001b[1;32mc:\\gitProjects\\yeni\\.venv\\lib\\site-packages\\nltk\\stem\\snowball.py:106\u001b[0m, in \u001b[0;36mSnowballStemmer.__init__\u001b[1;34m(self, language, ignore_stopwords)\u001b[0m\n\u001b[0;32m    104\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m__init__\u001b[39m(\u001b[38;5;28mself\u001b[39m, language, ignore_stopwords\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mFalse\u001b[39;00m):\n\u001b[0;32m    105\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m language \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mlanguages:\n\u001b[1;32m--> 106\u001b[0m         \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mThe language \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mlanguage\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m is not supported.\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[0;32m    107\u001b[0m     stemmerclass \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mglobals\u001b[39m()[language\u001b[38;5;241m.\u001b[39mcapitalize() \u001b[38;5;241m+\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mStemmer\u001b[39m\u001b[38;5;124m\"\u001b[39m]\n\u001b[0;32m    108\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mstemmer \u001b[38;5;241m=\u001b[39m stemmerclass(ignore_stopwords)\n",
      "\u001b[1;31mValueError\u001b[0m: The language 'turkish' is not supported."
     ]
    }
   ],
   "source": [
    "import re\n",
    "import pandas as pd\n",
    "from nltk.stem.snowball import SnowballStemmer\n",
    "\n",
    "# Türkçe stemmer\n",
    "stemmer = SnowballStemmer(\"turkish\")\n",
    "\n",
    "def stem_text(text):\n",
    "    words = text.split()\n",
    "    stemmed_words = [stemmer.stem(word) for word in words]\n",
    "    return ' '.join(stemmed_words)\n",
    "\n",
    "class DataProcessor:\n",
    "    def __init__(self, input_csv, output_csv, max_words=300, model_name='dbmdz/distilbert-base-turkish-cased'):\n",
    "        self.input_csv = input_csv\n",
    "        self.output_csv = output_csv\n",
    "        self.max_words = max_words\n",
    "        self.model_name = model_name\n",
    "\n",
    "    def main_pipeline(self):\n",
    "        def filter_text(text):\n",
    "            # Dış bağlantılar ve kaynakçaları kaldır\n",
    "            text = re.sub(r'http\\S+|https\\S+|\\b(?:www\\.)?\\S+\\.\\w{2,4}\\b', '', text)\n",
    "            # Tarih ve sayıları kaldır\n",
    "            text = re.sub(r'\\d{4}-\\d{2}-\\d{2}|\\d{2}/\\d{2}/\\d{4}|\\d+', '', text)\n",
    "            # Sayıları kaldır\n",
    "            text = re.sub(r'\\d+', '', text)\n",
    "            # Kısa veya uzun kelimeleri kaldır\n",
    "            words = text.split()\n",
    "            words = [word for word in words if 2 <= len(word) <= 20]\n",
    "            return ' '.join(words)\n",
    "        \n",
    "        # UTF-8 encoding ile dosyayı okuyun\n",
    "        df = pd.read_csv(self.input_csv, encoding='utf-8')\n",
    "\n",
    "        # Metinleri filtrele\n",
    "        df['filtered_text'] = df['metinler'].apply(filter_text)\n",
    "\n",
    "        # Filtrelenmiş metinleri stem (kök) yap\n",
    "        df['stemmed_text'] = df['filtered_text'].apply(stem_text)\n",
    "\n",
    "        # Sonuçları yeni bir CSV dosyasına kaydet\n",
    "        df.to_csv(self.output_csv, index=False, encoding='utf-8')\n",
    "\n",
    "# Kullanım örneği\n",
    "processor = DataProcessor(input_csv='texts_egitim.csv', output_csv='cleaned_data4.csv')\n",
    "processor.main_pipeline()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024-08-28 11:26:01 INFO: Checking for updates to resources.json in case models have been updated.  Note: this behavior can be turned off with download_method=None or download_method=DownloadMethod.REUSE_RESOURCES\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2af6508a7b6c48619ba0d6b3ce69e73b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading https://raw.githubusercontent.com/stanfordnlp/stanza-resources/main/resources_1.8.0.json:   0%|   …"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024-08-28 11:26:02 INFO: Downloaded file to C:\\Users\\info\\stanza_resources\\resources.json\n",
      "2024-08-28 11:26:02 INFO: Loading these models for language: tr (Turkish):\n",
      "=============================\n",
      "| Processor | Package       |\n",
      "-----------------------------\n",
      "| tokenize  | imst          |\n",
      "| mwt       | imst          |\n",
      "| pos       | imst_charlm   |\n",
      "| lemma     | imst_nocharlm |\n",
      "| ner       | starlang      |\n",
      "=============================\n",
      "\n",
      "2024-08-28 11:26:02 INFO: Using device: cpu\n",
      "2024-08-28 11:26:02 INFO: Loading: tokenize\n",
      "c:\\gitProjects\\yeni\\.venv\\lib\\site-packages\\stanza\\models\\tokenization\\trainer.py:82: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.\n",
      "  checkpoint = torch.load(filename, lambda storage, loc: storage)\n",
      "2024-08-28 11:26:02 INFO: Loading: mwt\n",
      "c:\\gitProjects\\yeni\\.venv\\lib\\site-packages\\stanza\\models\\mwt\\trainer.py:170: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.\n",
      "  checkpoint = torch.load(filename, lambda storage, loc: storage)\n",
      "2024-08-28 11:26:02 INFO: Loading: pos\n",
      "c:\\gitProjects\\yeni\\.venv\\lib\\site-packages\\stanza\\models\\pos\\trainer.py:139: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.\n",
      "  checkpoint = torch.load(filename, lambda storage, loc: storage)\n",
      "c:\\gitProjects\\yeni\\.venv\\lib\\site-packages\\stanza\\models\\common\\pretrain.py:56: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.\n",
      "  data = torch.load(self.filename, lambda storage, loc: storage)\n",
      "c:\\gitProjects\\yeni\\.venv\\lib\\site-packages\\stanza\\models\\common\\char_model.py:271: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.\n",
      "  state = torch.load(filename, lambda storage, loc: storage)\n",
      "2024-08-28 11:26:02 INFO: Loading: lemma\n",
      "c:\\gitProjects\\yeni\\.venv\\lib\\site-packages\\stanza\\models\\lemma\\trainer.py:236: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.\n",
      "  checkpoint = torch.load(filename, lambda storage, loc: storage)\n",
      "2024-08-28 11:26:02 INFO: Loading: ner\n",
      "c:\\gitProjects\\yeni\\.venv\\lib\\site-packages\\stanza\\models\\ner\\trainer.py:197: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.\n",
      "  checkpoint = torch.load(filename, lambda storage, loc: storage)\n",
      "2024-08-28 11:26:03 INFO: Done loading processors!\n"
     ]
    },
    {
     "ename": "",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31mThe Kernel crashed while executing code in the current cell or a previous cell. \n",
      "\u001b[1;31mPlease review the code in the cell(s) to identify a possible cause of the failure. \n",
      "\u001b[1;31mClick <a href='https://aka.ms/vscodeJupyterKernelCrash'>here</a> for more info. \n",
      "\u001b[1;31mView Jupyter <a href='command:jupyter.viewOutput'>log</a> for further details."
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import re\n",
    "from transformers import AutoTokenizer\n",
    "import spacy\n",
    "import stanza\n",
    "\n",
    "# ------------------------ Cümlelerin boyutlarını ve stop words'leri tanımladığımız yer -----------------------------\n",
    "nlp = stanza.Pipeline('tr', processors='tokenize,mwt,pos,lemma,ner')\n",
    "\n",
    "def preprocess_text(text, stopwords):\n",
    "    doc = nlp(text)\n",
    "    tokens = [\n",
    "        word.lemma if word.lemma is not None else word.text\n",
    "        for sentence in doc.sentences\n",
    "        for word in sentence.words\n",
    "        if word.text.lower() not in stopwords\n",
    "    ]\n",
    "    return \" \".join(tokens)\n",
    "\n",
    "def extract_keywords_and_subheadings(text):\n",
    "    doc = nlp(text)\n",
    "    keywords = []\n",
    "    subheadings = []\n",
    "    for ent in doc.ents:\n",
    "        if ent.label_ == \"ORG\" or ent.label_ == \"PERSON\":  # Örnek: Kurum veya kişi isimleri\n",
    "            keywords.append(ent.text)\n",
    "        elif ent.label_ == \"GPE\":  # Örnek: Yer isimleri\n",
    "            subheadings.append(ent.text)\n",
    "    return keywords, subheadings\n",
    "\n",
    "def truncate_text_meaningful(text, max_len=300):\n",
    "    doc = nlp(text)\n",
    "    tokens = [word.lemma if word.lemma is not None else word.text for word in doc.words]\n",
    "    truncated_text = ' '.join(tokens[:max_len])\n",
    "    return truncated_text\n",
    "\n",
    "def tokenize_and_pad(data, model_name='bert-base-uncased', max_length=512):\n",
    "    tokenizer = AutoTokenizer.from_pretrained(model_name)\n",
    "    encoded_input = tokenizer(data, padding=True, truncation=True, max_length=max_length)\n",
    "    return encoded_input\n",
    "\n",
    "class DataProcessor:\n",
    "    def __init__(self, input_csv, output_csv, stopword_file, max_words=300, model_name='dbmdz/distilbert-base-turkish-cased'):\n",
    "        self.input_csv = input_csv\n",
    "        self.output_csv = output_csv\n",
    "        self.stopword_file = stopword_file\n",
    "        self.max_words = max_words\n",
    "        self.model_name = model_name\n",
    "        self.stopwords = self.load_stopwords()\n",
    "\n",
    "    def load_stopwords(self):\n",
    "        with open(self.stopword_file, 'r', encoding='utf-8') as file:\n",
    "            stopwords = set(file.read().split())\n",
    "        return stopwords\n",
    "\n",
    "    def main_pipeline(self):\n",
    "        def filter_text(text):\n",
    "            text = re.sub(r'http\\S+|https\\S+|\\b(?:www\\.)?\\S+\\.\\w{2,4}\\b', '', text)\n",
    "            text = re.sub(r'\\d{4}-\\d{2}-\\d{2}|\\d{2}/\\d{2}/\\d{4}|\\d+', '', text)\n",
    "            text = re.sub(r'\\d+', '', text)\n",
    "            words = text.split()\n",
    "            words = [word for word in words if 2 <= len(word) <= 20]\n",
    "            return ' '.join(words)\n",
    "        \n",
    "        df = pd.read_csv(self.input_csv, encoding='utf-8')\n",
    "        \n",
    "        if 'metinler' not in df.columns:\n",
    "            raise ValueError(\"CSV dosyasında 'metinler' adlı bir sütun bulunamadı. Lütfen sütun adını kontrol edin.\")\n",
    "        \n",
    "        df['kısaltılmıs_metin'] = df['metinler'].apply(lambda x: preprocess_text(x, self.stopwords))\n",
    "        df['kısaltılmıs_metin'] = df['kısaltılmıs_metin'].apply(lambda x: truncate_text_meaningful(x, max_len=self.max_words))\n",
    "\n",
    "        padded_tokens = tokenize_and_pad(df['kısaltılmıs_metin'].tolist(), model_name=self.model_name)\n",
    "        df['padded_tokens'] = padded_tokens['input_ids']\n",
    "\n",
    "        print(\"Kısaltılmış metinler:\")\n",
    "        print(df['kısaltılmıs_metin'].head())\n",
    "        print(\"Tokenize edilmiş ve padding uygulanmış veriler:\")\n",
    "        print(df[['kısaltılmıs_metin', 'padded_tokens']].head())\n",
    "\n",
    "        self.save_cleaned_data(df)\n",
    "    \n",
    "    def save_cleaned_data(self, df):\n",
    "        df.to_csv(self.output_csv, index=False, encoding='utf-8')\n",
    "        print(f\"Temizlenmiş veri '{self.output_csv}' dosyasına kaydedildi.\")\n",
    "\n",
    "processor = DataProcessor(input_csv=\"texts_egitim.csv\", output_csv=\"cleaned_data4.csv\", stopword_file=\"gereksiz_kelimeler.txt\")\n",
    "processor.main_pipeline()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from transformers import AutoModel, AutoTokenizer\n",
    "import torch\n",
    "from sklearn.metrics.pairwise import cosine_similarity\n",
    "\n",
    "# Model ve tokenizer'ı yükleme\n",
    "model_name = 'dbmdz/distilbert-base-turkish-cased'\n",
    "tokenizer = AutoTokenizer.from_pretrained(model_name)\n",
    "model = AutoModel.from_pretrained(model_name)\n",
    "\n",
    "def get_embeddings(text):\n",
    "    inputs = tokenizer(text, return_tensors='pt', truncation=True, padding=True)\n",
    "    with torch.no_grad():\n",
    "        outputs = model(**inputs)\n",
    "    # İlk özniteliklerin ortalamasını alarak vektör elde etme\n",
    "    embeddings = outputs.last_hidden_state.mean(dim=1).squeeze()\n",
    "    return embeddings\n",
    "\n",
    "def generate_subheadings(title, keywords, top_n=5):\n",
    "    title_embedding = get_embeddings(title)\n",
    "    keyword_embeddings = [get_embeddings(keyword) for keyword in keywords]\n",
    "\n",
    "    similarities = [cosine_similarity(title_embedding.unsqueeze(0), keyword_embedding.unsqueeze(0))[0][0] for keyword_embedding in keyword_embeddings]\n",
    "    sorted_indices = sorted(range(len(similarities)), key=lambda i: similarities[i], reverse=True)\n",
    "\n",
    "    # Benzerliklere göre anahtar kelimelerden alt başlıklar oluşturma\n",
    "    subheadings = [keywords[i] for i in sorted_indices[:top_n]]\n",
    "    return subheadings\n",
    "\n",
    "# Örnek veri\n",
    "title = \"Veri Bilimi Nedir?\"\n",
    "keywords = [\"Makine Öğrenmesi\", \"Büyük Veri\", \"Yapay Zeka\", \"1936\", \"Veri Madenciliği\", \"Derin Öğrenme\", \"Doğal Dil İşleme\", \"1982\", \"Yapay Sinir Ağları\", \"Kümeleme\"]\n",
    "\n",
    "# Alt başlıkları oluşturma\n",
    "subheadings = generate_subheadings(title, keywords)\n",
    "print(\"Alt Başlıklar:\")\n",
    "for subheading in subheadings:\n",
    "    print(f\"- {subheading}\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                                        kısaltılmıs_metin\n",
      "0       W F 295–596 P N 82 3–211 78 pp 137–146 5–537 \"...\n",
      "1       Pasaportunun Ama 1934 Balıkesir'de Doktoru Pro...\n",
      "2       hesaplamalar ortalarına 2 II Bu kümesi İlk II ...\n",
      "3       1 20 14 19 12 14 Yazın Türkçedir Zaman geçirmi...\n",
      "4       \" ) inşasıyla ilgilenir \"Ters mühendislik, müh...\n",
      "...                                                   ...\n",
      "104103  Xenocicerina, Cicerininae altfamilyasına cinsi...\n",
      "104104  Paracicerina, Cicerininae altfamilyasına cinsi...\n",
      "104105  Lig futbolcuları Yasin Güreler (d Lig futbolcu...\n",
      "104106  Elvertia, Kalyptorhynchia seksiyonuna cinsidir...\n",
      "104107  Kaynakça Ek Hartcher New York: HarperCollins M...\n",
      "\n",
      "[104108 rows x 1 columns]\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>kısaltılmıs_metin</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>W F 295–596 P N 82 3–211 78 pp 137–146 5–537 \"...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Pasaportunun Ama 1934 Balıkesir'de Doktoru Pro...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>hesaplamalar ortalarına 2 II Bu kümesi İlk II ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1 20 14 19 12 14 Yazın Türkçedir Zaman geçirmi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>\" ) inşasıyla ilgilenir \"Ters mühendislik, müh...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>104103</th>\n",
       "      <td>Xenocicerina, Cicerininae altfamilyasına cinsi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>104104</th>\n",
       "      <td>Paracicerina, Cicerininae altfamilyasına cinsi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>104105</th>\n",
       "      <td>Lig futbolcuları Yasin Güreler (d Lig futbolcu...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>104106</th>\n",
       "      <td>Elvertia, Kalyptorhynchia seksiyonuna cinsidir...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>104107</th>\n",
       "      <td>Kaynakça Ek Hartcher New York: HarperCollins M...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>104108 rows × 1 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                        kısaltılmıs_metin\n",
       "0       W F 295–596 P N 82 3–211 78 pp 137–146 5–537 \"...\n",
       "1       Pasaportunun Ama 1934 Balıkesir'de Doktoru Pro...\n",
       "2       hesaplamalar ortalarına 2 II Bu kümesi İlk II ...\n",
       "3       1 20 14 19 12 14 Yazın Türkçedir Zaman geçirmi...\n",
       "4       \" ) inşasıyla ilgilenir \"Ters mühendislik, müh...\n",
       "...                                                   ...\n",
       "104103  Xenocicerina, Cicerininae altfamilyasına cinsi...\n",
       "104104  Paracicerina, Cicerininae altfamilyasına cinsi...\n",
       "104105  Lig futbolcuları Yasin Güreler (d Lig futbolcu...\n",
       "104106  Elvertia, Kalyptorhynchia seksiyonuna cinsidir...\n",
       "104107  Kaynakça Ek Hartcher New York: HarperCollins M...\n",
       "\n",
       "[104108 rows x 1 columns]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "\n",
    "# CSV dosyasını oku\n",
    "df = pd.read_csv('cleaned_data.csv')\n",
    "\n",
    "# Görmek istediğiniz üç sütunu seçin\n",
    "selected_columns = df[['kısaltılmıs_metin']]\n",
    "\n",
    "# Seçilen sütunları tablo olarak görüntüle\n",
    "print(selected_columns)\n",
    "\n",
    "# Eğer Jupyter Notebook kullanıyorsanız, daha güzel görüntü için display() fonksiyonunu kullanabilirsiniz:\n",
    "from IPython.display import display\n",
    "display(selected_columns)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                                            padded_tokens\n",
      "0       [2, 59, 42, 4550, 1092, 550, 8528, 1062, 52, 5...\n",
      "1       [2, 7722, 11428, 2297, 2742, 7395, 1119, 10491...\n",
      "2       [2, 16306, 1980, 3008, 2431, 22, 6477, 2123, 2...\n",
      "3       [2, 21, 2146, 3226, 2401, 2836, 3226, 27718, 5...\n",
      "4       [2, 6, 13, 29132, 2218, 5999, 1977, 6, 24444, ...\n",
      "...                                                   ...\n",
      "104103  [2, 60, 1975, 2370, 6546, 4689, 1006, 16, 39, ...\n",
      "104104  [2, 8149, 2329, 5790, 3930, 16, 39, 6546, 4689...\n",
      "104105  [2, 5379, 17315, 1048, 19661, 5646, 2070, 12, ...\n",
      "104106  [2, 3026, 11411, 4475, 16, 3771, 1032, 6728, 1...\n",
      "104107  [2, 7934, 2548, 2951, 3698, 1023, 14059, 5510,...\n",
      "\n",
      "[104108 rows x 1 columns]\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>padded_tokens</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>[2, 59, 42, 4550, 1092, 550, 8528, 1062, 52, 5...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>[2, 7722, 11428, 2297, 2742, 7395, 1119, 10491...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>[2, 16306, 1980, 3008, 2431, 22, 6477, 2123, 2...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>[2, 21, 2146, 3226, 2401, 2836, 3226, 27718, 5...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>[2, 6, 13, 29132, 2218, 5999, 1977, 6, 24444, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>104103</th>\n",
       "      <td>[2, 60, 1975, 2370, 6546, 4689, 1006, 16, 39, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>104104</th>\n",
       "      <td>[2, 8149, 2329, 5790, 3930, 16, 39, 6546, 4689...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>104105</th>\n",
       "      <td>[2, 5379, 17315, 1048, 19661, 5646, 2070, 12, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>104106</th>\n",
       "      <td>[2, 3026, 11411, 4475, 16, 3771, 1032, 6728, 1...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>104107</th>\n",
       "      <td>[2, 7934, 2548, 2951, 3698, 1023, 14059, 5510,...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>104108 rows × 1 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                            padded_tokens\n",
       "0       [2, 59, 42, 4550, 1092, 550, 8528, 1062, 52, 5...\n",
       "1       [2, 7722, 11428, 2297, 2742, 7395, 1119, 10491...\n",
       "2       [2, 16306, 1980, 3008, 2431, 22, 6477, 2123, 2...\n",
       "3       [2, 21, 2146, 3226, 2401, 2836, 3226, 27718, 5...\n",
       "4       [2, 6, 13, 29132, 2218, 5999, 1977, 6, 24444, ...\n",
       "...                                                   ...\n",
       "104103  [2, 60, 1975, 2370, 6546, 4689, 1006, 16, 39, ...\n",
       "104104  [2, 8149, 2329, 5790, 3930, 16, 39, 6546, 4689...\n",
       "104105  [2, 5379, 17315, 1048, 19661, 5646, 2070, 12, ...\n",
       "104106  [2, 3026, 11411, 4475, 16, 3771, 1032, 6728, 1...\n",
       "104107  [2, 7934, 2548, 2951, 3698, 1023, 14059, 5510,...\n",
       "\n",
       "[104108 rows x 1 columns]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "\n",
    "# CSV dosyasını oku\n",
    "df = pd.read_csv('cleaned_data.csv')\n",
    "\n",
    "# Görmek istediğiniz üç sütunu seçin\n",
    "selected_columns = df[['padded_tokens']]\n",
    "\n",
    "# Seçilen sütunları tablo olarak görüntüle\n",
    "print(selected_columns)\n",
    "\n",
    "# Eğer Jupyter Notebook kullanıyorsanız, daha güzel görüntü için display() fonksiyonunu kullanabilirsiniz:\n",
    "from IPython.display import display\n",
    "display(selected_columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from gensim import corpora\n",
    "from gensim.models import LdaMulticore\n",
    "import pandas as pd\n",
    "\n",
    "# CSV dosyasını okuma\n",
    "df = pd.read_csv('cleaned_data3.csv')\n",
    "\n",
    "# Verinin bir alt kümesini seçme\n",
    "df_sample = df.sample(n=10000, random_state=100)\n",
    "\n",
    "# Kelimeleri token'lara ayırma\n",
    "tokenized_text = [text.split() for text in df_sample['kısaltılmıs_metin']]\n",
    "\n",
    "# Dictionary ve Corpus oluşturma\n",
    "id2word = corpora.Dictionary(tokenized_text)\n",
    "corpus = [id2word.doc2bow(text) for text in tokenized_text]\n",
    "\n",
    "# LDA Modelini Eğitme\n",
    "lda_model = LdaMulticore(\n",
    "    corpus=corpus,\n",
    "    id2word=id2word,\n",
    "    num_topics=5,\n",
    "    random_state=100,\n",
    "    chunksize=50,\n",
    "    passes=5,\n",
    "    alpha='symmetric',\n",
    "    eta='auto',\n",
    "    per_word_topics=True,\n",
    "    workers=4  # Paralel iş parçacıkları kullanarak performansı artırır\n",
    ")\n",
    "\n",
    "# Sonuçları görüntüleme\n",
    "for idx, topic in lda_model.print_topics(-1):\n",
    "    print(f\"Topic: {idx}\\nWords: {topic}\\n\")\n",
    "\n",
    "# Alt kümesini kaydetme\n",
    "df_sample.to_csv('cleaned_processed_data.csv', index=False)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Subheadings belirleme "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\gitProjects\\yeni\\.venv\\lib\\site-packages\\sentence_transformers\\cross_encoder\\CrossEncoder.py:11: TqdmExperimentalWarning: Using `tqdm.autonotebook.tqdm` in notebook mode. Use `tqdm.tqdm` instead to force console mode (e.g. in jupyter console)\n",
      "  from tqdm.autonotebook import tqdm, trange\n",
      "c:\\gitProjects\\yeni\\.venv\\lib\\site-packages\\transformers\\utils\\generic.py:441: FutureWarning: `torch.utils._pytree._register_pytree_node` is deprecated. Please use `torch.utils._pytree.register_pytree_node` instead.\n",
      "  _torch_pytree._register_pytree_node(\n",
      "c:\\gitProjects\\yeni\\.venv\\lib\\site-packages\\transformers\\utils\\generic.py:309: FutureWarning: `torch.utils._pytree._register_pytree_node` is deprecated. Please use `torch.utils._pytree.register_pytree_node` instead.\n",
      "  _torch_pytree._register_pytree_node(\n",
      "c:\\gitProjects\\yeni\\.venv\\lib\\site-packages\\huggingface_hub\\file_download.py:1150: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n",
      "  warnings.warn(\n",
      "c:\\gitProjects\\yeni\\.venv\\lib\\site-packages\\transformers\\utils\\generic.py:309: FutureWarning: `torch.utils._pytree._register_pytree_node` is deprecated. Please use `torch.utils._pytree.register_pytree_node` instead.\n",
      "  _torch_pytree._register_pytree_node(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Batch 1 işlenip kaydedildi.\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "Cell \u001b[1;32mIn[1], line 71\u001b[0m\n\u001b[0;32m     68\u001b[0m     \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mSubheadings başarıyla \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124msubheadings.csv\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m dosyasına kaydedildi.\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[0;32m     70\u001b[0m \u001b[38;5;66;03m# CSV'ye yazma işlemini başlatma\u001b[39;00m\n\u001b[1;32m---> 71\u001b[0m \u001b[43mprocess_documents\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n",
      "Cell \u001b[1;32mIn[1], line 54\u001b[0m, in \u001b[0;36mprocess_documents\u001b[1;34m(batch_size, top_n_subheadings)\u001b[0m\n\u001b[0;32m     51\u001b[0m keywords \u001b[38;5;241m=\u001b[39m doc[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mkeywords\u001b[39m\u001b[38;5;124m'\u001b[39m]\n\u001b[0;32m     52\u001b[0m text \u001b[38;5;241m=\u001b[39m doc[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mtext\u001b[39m\u001b[38;5;124m'\u001b[39m]\n\u001b[1;32m---> 54\u001b[0m top_subheadings \u001b[38;5;241m=\u001b[39m \u001b[43mget_top_subheadings\u001b[49m\u001b[43m(\u001b[49m\u001b[43mtitle\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mkeywords\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtext\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m     56\u001b[0m \u001b[38;5;66;03m# Verileri listeye ekleyin\u001b[39;00m\n\u001b[0;32m     57\u001b[0m data\u001b[38;5;241m.\u001b[39mappend({\n\u001b[0;32m     58\u001b[0m     \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mTitle\u001b[39m\u001b[38;5;124m'\u001b[39m: title,\n\u001b[0;32m     59\u001b[0m     \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mKeywords\u001b[39m\u001b[38;5;124m'\u001b[39m: \u001b[38;5;124m'\u001b[39m\u001b[38;5;124m, \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;241m.\u001b[39mjoin(keywords),\n\u001b[0;32m     60\u001b[0m     \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mSubheadings\u001b[39m\u001b[38;5;124m'\u001b[39m: \u001b[38;5;124m'\u001b[39m\u001b[38;5;124m; \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;241m.\u001b[39mjoin(top_subheadings)\n\u001b[0;32m     61\u001b[0m })\n",
      "Cell \u001b[1;32mIn[1], line 32\u001b[0m, in \u001b[0;36mget_top_subheadings\u001b[1;34m(title, keywords, text, top_n)\u001b[0m\n\u001b[0;32m     30\u001b[0m keywords_embedding \u001b[38;5;241m=\u001b[39m embed_text(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;241m.\u001b[39mjoin(keywords))\n\u001b[0;32m     31\u001b[0m subheadings \u001b[38;5;241m=\u001b[39m extract_subheadings(text)\n\u001b[1;32m---> 32\u001b[0m subheadings_embeddings \u001b[38;5;241m=\u001b[39m [embed_text(sub) \u001b[38;5;28;01mfor\u001b[39;00m sub \u001b[38;5;129;01min\u001b[39;00m subheadings]\n\u001b[0;32m     34\u001b[0m scores \u001b[38;5;241m=\u001b[39m [cosine_similarity([title_embedding], [embedding])[\u001b[38;5;241m0\u001b[39m][\u001b[38;5;241m0\u001b[39m] \u001b[38;5;241m+\u001b[39m \n\u001b[0;32m     35\u001b[0m           cosine_similarity([keywords_embedding], [embedding])[\u001b[38;5;241m0\u001b[39m][\u001b[38;5;241m0\u001b[39m]\n\u001b[0;32m     36\u001b[0m           \u001b[38;5;28;01mfor\u001b[39;00m embedding \u001b[38;5;129;01min\u001b[39;00m subheadings_embeddings]\n\u001b[0;32m     38\u001b[0m top_indices \u001b[38;5;241m=\u001b[39m np\u001b[38;5;241m.\u001b[39margsort(scores)[\u001b[38;5;241m-\u001b[39mtop_n:]\n",
      "Cell \u001b[1;32mIn[1], line 32\u001b[0m, in \u001b[0;36m<listcomp>\u001b[1;34m(.0)\u001b[0m\n\u001b[0;32m     30\u001b[0m keywords_embedding \u001b[38;5;241m=\u001b[39m embed_text(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;241m.\u001b[39mjoin(keywords))\n\u001b[0;32m     31\u001b[0m subheadings \u001b[38;5;241m=\u001b[39m extract_subheadings(text)\n\u001b[1;32m---> 32\u001b[0m subheadings_embeddings \u001b[38;5;241m=\u001b[39m [\u001b[43membed_text\u001b[49m\u001b[43m(\u001b[49m\u001b[43msub\u001b[49m\u001b[43m)\u001b[49m \u001b[38;5;28;01mfor\u001b[39;00m sub \u001b[38;5;129;01min\u001b[39;00m subheadings]\n\u001b[0;32m     34\u001b[0m scores \u001b[38;5;241m=\u001b[39m [cosine_similarity([title_embedding], [embedding])[\u001b[38;5;241m0\u001b[39m][\u001b[38;5;241m0\u001b[39m] \u001b[38;5;241m+\u001b[39m \n\u001b[0;32m     35\u001b[0m           cosine_similarity([keywords_embedding], [embedding])[\u001b[38;5;241m0\u001b[39m][\u001b[38;5;241m0\u001b[39m]\n\u001b[0;32m     36\u001b[0m           \u001b[38;5;28;01mfor\u001b[39;00m embedding \u001b[38;5;129;01min\u001b[39;00m subheadings_embeddings]\n\u001b[0;32m     38\u001b[0m top_indices \u001b[38;5;241m=\u001b[39m np\u001b[38;5;241m.\u001b[39margsort(scores)[\u001b[38;5;241m-\u001b[39mtop_n:]\n",
      "Cell \u001b[1;32mIn[1], line 25\u001b[0m, in \u001b[0;36membed_text\u001b[1;34m(text)\u001b[0m\n\u001b[0;32m     24\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21membed_text\u001b[39m(text):\n\u001b[1;32m---> 25\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mmodel\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mencode\u001b[49m\u001b[43m(\u001b[49m\u001b[43mtext\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[1;32mc:\\gitProjects\\yeni\\.venv\\lib\\site-packages\\sentence_transformers\\SentenceTransformer.py:517\u001b[0m, in \u001b[0;36mSentenceTransformer.encode\u001b[1;34m(self, sentences, prompt_name, prompt, batch_size, show_progress_bar, output_value, precision, convert_to_numpy, convert_to_tensor, device, normalize_embeddings)\u001b[0m\n\u001b[0;32m    514\u001b[0m features\u001b[38;5;241m.\u001b[39mupdate(extra_features)\n\u001b[0;32m    516\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m torch\u001b[38;5;241m.\u001b[39mno_grad():\n\u001b[1;32m--> 517\u001b[0m     out_features \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mforward\u001b[49m\u001b[43m(\u001b[49m\u001b[43mfeatures\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    518\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdevice\u001b[38;5;241m.\u001b[39mtype \u001b[38;5;241m==\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mhpu\u001b[39m\u001b[38;5;124m\"\u001b[39m:\n\u001b[0;32m    519\u001b[0m         out_features \u001b[38;5;241m=\u001b[39m copy\u001b[38;5;241m.\u001b[39mdeepcopy(out_features)\n",
      "File \u001b[1;32mc:\\gitProjects\\yeni\\.venv\\lib\\site-packages\\torch\\nn\\modules\\container.py:219\u001b[0m, in \u001b[0;36mSequential.forward\u001b[1;34m(self, input)\u001b[0m\n\u001b[0;32m    217\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mforward\u001b[39m(\u001b[38;5;28mself\u001b[39m, \u001b[38;5;28minput\u001b[39m):\n\u001b[0;32m    218\u001b[0m     \u001b[38;5;28;01mfor\u001b[39;00m module \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m:\n\u001b[1;32m--> 219\u001b[0m         \u001b[38;5;28minput\u001b[39m \u001b[38;5;241m=\u001b[39m \u001b[43mmodule\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43minput\u001b[39;49m\u001b[43m)\u001b[49m\n\u001b[0;32m    220\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28minput\u001b[39m\n",
      "File \u001b[1;32mc:\\gitProjects\\yeni\\.venv\\lib\\site-packages\\torch\\nn\\modules\\module.py:1553\u001b[0m, in \u001b[0;36mModule._wrapped_call_impl\u001b[1;34m(self, *args, **kwargs)\u001b[0m\n\u001b[0;32m   1551\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_compiled_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)  \u001b[38;5;66;03m# type: ignore[misc]\u001b[39;00m\n\u001b[0;32m   1552\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m-> 1553\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n",
      "File \u001b[1;32mc:\\gitProjects\\yeni\\.venv\\lib\\site-packages\\torch\\nn\\modules\\module.py:1562\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[1;34m(self, *args, **kwargs)\u001b[0m\n\u001b[0;32m   1557\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[0;32m   1558\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[0;32m   1559\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[0;32m   1560\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[0;32m   1561\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[1;32m-> 1562\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m forward_call(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n\u001b[0;32m   1564\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m   1565\u001b[0m     result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n",
      "File \u001b[1;32mc:\\gitProjects\\yeni\\.venv\\lib\\site-packages\\sentence_transformers\\models\\Transformer.py:118\u001b[0m, in \u001b[0;36mTransformer.forward\u001b[1;34m(self, features)\u001b[0m\n\u001b[0;32m    115\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mtoken_type_ids\u001b[39m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;129;01min\u001b[39;00m features:\n\u001b[0;32m    116\u001b[0m     trans_features[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mtoken_type_ids\u001b[39m\u001b[38;5;124m\"\u001b[39m] \u001b[38;5;241m=\u001b[39m features[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mtoken_type_ids\u001b[39m\u001b[38;5;124m\"\u001b[39m]\n\u001b[1;32m--> 118\u001b[0m output_states \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mauto_model(\u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mtrans_features, return_dict\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mFalse\u001b[39;00m)\n\u001b[0;32m    119\u001b[0m output_tokens \u001b[38;5;241m=\u001b[39m output_states[\u001b[38;5;241m0\u001b[39m]\n\u001b[0;32m    121\u001b[0m features\u001b[38;5;241m.\u001b[39mupdate({\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mtoken_embeddings\u001b[39m\u001b[38;5;124m\"\u001b[39m: output_tokens, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mattention_mask\u001b[39m\u001b[38;5;124m\"\u001b[39m: features[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mattention_mask\u001b[39m\u001b[38;5;124m\"\u001b[39m]})\n",
      "File \u001b[1;32mc:\\gitProjects\\yeni\\.venv\\lib\\site-packages\\torch\\nn\\modules\\module.py:1553\u001b[0m, in \u001b[0;36mModule._wrapped_call_impl\u001b[1;34m(self, *args, **kwargs)\u001b[0m\n\u001b[0;32m   1551\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_compiled_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)  \u001b[38;5;66;03m# type: ignore[misc]\u001b[39;00m\n\u001b[0;32m   1552\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m-> 1553\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n",
      "File \u001b[1;32mc:\\gitProjects\\yeni\\.venv\\lib\\site-packages\\torch\\nn\\modules\\module.py:1562\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[1;34m(self, *args, **kwargs)\u001b[0m\n\u001b[0;32m   1557\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[0;32m   1558\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[0;32m   1559\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[0;32m   1560\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[0;32m   1561\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[1;32m-> 1562\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m forward_call(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n\u001b[0;32m   1564\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m   1565\u001b[0m     result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n",
      "File \u001b[1;32mc:\\gitProjects\\yeni\\.venv\\lib\\site-packages\\transformers\\models\\bert\\modeling_bert.py:1013\u001b[0m, in \u001b[0;36mBertModel.forward\u001b[1;34m(self, input_ids, attention_mask, token_type_ids, position_ids, head_mask, inputs_embeds, encoder_hidden_states, encoder_attention_mask, past_key_values, use_cache, output_attentions, output_hidden_states, return_dict)\u001b[0m\n\u001b[0;32m   1004\u001b[0m head_mask \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mget_head_mask(head_mask, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mconfig\u001b[38;5;241m.\u001b[39mnum_hidden_layers)\n\u001b[0;32m   1006\u001b[0m embedding_output \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39membeddings(\n\u001b[0;32m   1007\u001b[0m     input_ids\u001b[38;5;241m=\u001b[39minput_ids,\n\u001b[0;32m   1008\u001b[0m     position_ids\u001b[38;5;241m=\u001b[39mposition_ids,\n\u001b[1;32m   (...)\u001b[0m\n\u001b[0;32m   1011\u001b[0m     past_key_values_length\u001b[38;5;241m=\u001b[39mpast_key_values_length,\n\u001b[0;32m   1012\u001b[0m )\n\u001b[1;32m-> 1013\u001b[0m encoder_outputs \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mencoder\u001b[49m\u001b[43m(\u001b[49m\n\u001b[0;32m   1014\u001b[0m \u001b[43m    \u001b[49m\u001b[43membedding_output\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m   1015\u001b[0m \u001b[43m    \u001b[49m\u001b[43mattention_mask\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mextended_attention_mask\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m   1016\u001b[0m \u001b[43m    \u001b[49m\u001b[43mhead_mask\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mhead_mask\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m   1017\u001b[0m \u001b[43m    \u001b[49m\u001b[43mencoder_hidden_states\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mencoder_hidden_states\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m   1018\u001b[0m \u001b[43m    \u001b[49m\u001b[43mencoder_attention_mask\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mencoder_extended_attention_mask\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m   1019\u001b[0m \u001b[43m    \u001b[49m\u001b[43mpast_key_values\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mpast_key_values\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m   1020\u001b[0m \u001b[43m    \u001b[49m\u001b[43muse_cache\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43muse_cache\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m   1021\u001b[0m \u001b[43m    \u001b[49m\u001b[43moutput_attentions\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43moutput_attentions\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m   1022\u001b[0m \u001b[43m    \u001b[49m\u001b[43moutput_hidden_states\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43moutput_hidden_states\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m   1023\u001b[0m \u001b[43m    \u001b[49m\u001b[43mreturn_dict\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mreturn_dict\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m   1024\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m   1025\u001b[0m sequence_output \u001b[38;5;241m=\u001b[39m encoder_outputs[\u001b[38;5;241m0\u001b[39m]\n\u001b[0;32m   1026\u001b[0m pooled_output \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mpooler(sequence_output) \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mpooler \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;28;01melse\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m\n",
      "File \u001b[1;32mc:\\gitProjects\\yeni\\.venv\\lib\\site-packages\\torch\\nn\\modules\\module.py:1553\u001b[0m, in \u001b[0;36mModule._wrapped_call_impl\u001b[1;34m(self, *args, **kwargs)\u001b[0m\n\u001b[0;32m   1551\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_compiled_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)  \u001b[38;5;66;03m# type: ignore[misc]\u001b[39;00m\n\u001b[0;32m   1552\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m-> 1553\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n",
      "File \u001b[1;32mc:\\gitProjects\\yeni\\.venv\\lib\\site-packages\\torch\\nn\\modules\\module.py:1562\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[1;34m(self, *args, **kwargs)\u001b[0m\n\u001b[0;32m   1557\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[0;32m   1558\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[0;32m   1559\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[0;32m   1560\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[0;32m   1561\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[1;32m-> 1562\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m forward_call(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n\u001b[0;32m   1564\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m   1565\u001b[0m     result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n",
      "File \u001b[1;32mc:\\gitProjects\\yeni\\.venv\\lib\\site-packages\\transformers\\models\\bert\\modeling_bert.py:607\u001b[0m, in \u001b[0;36mBertEncoder.forward\u001b[1;34m(self, hidden_states, attention_mask, head_mask, encoder_hidden_states, encoder_attention_mask, past_key_values, use_cache, output_attentions, output_hidden_states, return_dict)\u001b[0m\n\u001b[0;32m    596\u001b[0m     layer_outputs \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_gradient_checkpointing_func(\n\u001b[0;32m    597\u001b[0m         layer_module\u001b[38;5;241m.\u001b[39m\u001b[38;5;21m__call__\u001b[39m,\n\u001b[0;32m    598\u001b[0m         hidden_states,\n\u001b[1;32m   (...)\u001b[0m\n\u001b[0;32m    604\u001b[0m         output_attentions,\n\u001b[0;32m    605\u001b[0m     )\n\u001b[0;32m    606\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m--> 607\u001b[0m     layer_outputs \u001b[38;5;241m=\u001b[39m \u001b[43mlayer_module\u001b[49m\u001b[43m(\u001b[49m\n\u001b[0;32m    608\u001b[0m \u001b[43m        \u001b[49m\u001b[43mhidden_states\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    609\u001b[0m \u001b[43m        \u001b[49m\u001b[43mattention_mask\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    610\u001b[0m \u001b[43m        \u001b[49m\u001b[43mlayer_head_mask\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    611\u001b[0m \u001b[43m        \u001b[49m\u001b[43mencoder_hidden_states\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    612\u001b[0m \u001b[43m        \u001b[49m\u001b[43mencoder_attention_mask\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    613\u001b[0m \u001b[43m        \u001b[49m\u001b[43mpast_key_value\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    614\u001b[0m \u001b[43m        \u001b[49m\u001b[43moutput_attentions\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    615\u001b[0m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    617\u001b[0m hidden_states \u001b[38;5;241m=\u001b[39m layer_outputs[\u001b[38;5;241m0\u001b[39m]\n\u001b[0;32m    618\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m use_cache:\n",
      "File \u001b[1;32mc:\\gitProjects\\yeni\\.venv\\lib\\site-packages\\torch\\nn\\modules\\module.py:1553\u001b[0m, in \u001b[0;36mModule._wrapped_call_impl\u001b[1;34m(self, *args, **kwargs)\u001b[0m\n\u001b[0;32m   1551\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_compiled_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)  \u001b[38;5;66;03m# type: ignore[misc]\u001b[39;00m\n\u001b[0;32m   1552\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m-> 1553\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n",
      "File \u001b[1;32mc:\\gitProjects\\yeni\\.venv\\lib\\site-packages\\torch\\nn\\modules\\module.py:1562\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[1;34m(self, *args, **kwargs)\u001b[0m\n\u001b[0;32m   1557\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[0;32m   1558\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[0;32m   1559\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[0;32m   1560\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[0;32m   1561\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[1;32m-> 1562\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m forward_call(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n\u001b[0;32m   1564\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m   1565\u001b[0m     result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n",
      "File \u001b[1;32mc:\\gitProjects\\yeni\\.venv\\lib\\site-packages\\transformers\\models\\bert\\modeling_bert.py:497\u001b[0m, in \u001b[0;36mBertLayer.forward\u001b[1;34m(self, hidden_states, attention_mask, head_mask, encoder_hidden_states, encoder_attention_mask, past_key_value, output_attentions)\u001b[0m\n\u001b[0;32m    485\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mforward\u001b[39m(\n\u001b[0;32m    486\u001b[0m     \u001b[38;5;28mself\u001b[39m,\n\u001b[0;32m    487\u001b[0m     hidden_states: torch\u001b[38;5;241m.\u001b[39mTensor,\n\u001b[1;32m   (...)\u001b[0m\n\u001b[0;32m    494\u001b[0m ) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m Tuple[torch\u001b[38;5;241m.\u001b[39mTensor]:\n\u001b[0;32m    495\u001b[0m     \u001b[38;5;66;03m# decoder uni-directional self-attention cached key/values tuple is at positions 1,2\u001b[39;00m\n\u001b[0;32m    496\u001b[0m     self_attn_past_key_value \u001b[38;5;241m=\u001b[39m past_key_value[:\u001b[38;5;241m2\u001b[39m] \u001b[38;5;28;01mif\u001b[39;00m past_key_value \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;28;01melse\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[1;32m--> 497\u001b[0m     self_attention_outputs \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mattention\u001b[49m\u001b[43m(\u001b[49m\n\u001b[0;32m    498\u001b[0m \u001b[43m        \u001b[49m\u001b[43mhidden_states\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    499\u001b[0m \u001b[43m        \u001b[49m\u001b[43mattention_mask\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    500\u001b[0m \u001b[43m        \u001b[49m\u001b[43mhead_mask\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    501\u001b[0m \u001b[43m        \u001b[49m\u001b[43moutput_attentions\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43moutput_attentions\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    502\u001b[0m \u001b[43m        \u001b[49m\u001b[43mpast_key_value\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mself_attn_past_key_value\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    503\u001b[0m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    504\u001b[0m     attention_output \u001b[38;5;241m=\u001b[39m self_attention_outputs[\u001b[38;5;241m0\u001b[39m]\n\u001b[0;32m    506\u001b[0m     \u001b[38;5;66;03m# if decoder, the last output is tuple of self-attn cache\u001b[39;00m\n",
      "File \u001b[1;32mc:\\gitProjects\\yeni\\.venv\\lib\\site-packages\\torch\\nn\\modules\\module.py:1553\u001b[0m, in \u001b[0;36mModule._wrapped_call_impl\u001b[1;34m(self, *args, **kwargs)\u001b[0m\n\u001b[0;32m   1551\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_compiled_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)  \u001b[38;5;66;03m# type: ignore[misc]\u001b[39;00m\n\u001b[0;32m   1552\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m-> 1553\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n",
      "File \u001b[1;32mc:\\gitProjects\\yeni\\.venv\\lib\\site-packages\\torch\\nn\\modules\\module.py:1562\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[1;34m(self, *args, **kwargs)\u001b[0m\n\u001b[0;32m   1557\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[0;32m   1558\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[0;32m   1559\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[0;32m   1560\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[0;32m   1561\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[1;32m-> 1562\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m forward_call(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n\u001b[0;32m   1564\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m   1565\u001b[0m     result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n",
      "File \u001b[1;32mc:\\gitProjects\\yeni\\.venv\\lib\\site-packages\\transformers\\models\\bert\\modeling_bert.py:427\u001b[0m, in \u001b[0;36mBertAttention.forward\u001b[1;34m(self, hidden_states, attention_mask, head_mask, encoder_hidden_states, encoder_attention_mask, past_key_value, output_attentions)\u001b[0m\n\u001b[0;32m    417\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mforward\u001b[39m(\n\u001b[0;32m    418\u001b[0m     \u001b[38;5;28mself\u001b[39m,\n\u001b[0;32m    419\u001b[0m     hidden_states: torch\u001b[38;5;241m.\u001b[39mTensor,\n\u001b[1;32m   (...)\u001b[0m\n\u001b[0;32m    425\u001b[0m     output_attentions: Optional[\u001b[38;5;28mbool\u001b[39m] \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mFalse\u001b[39;00m,\n\u001b[0;32m    426\u001b[0m ) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m Tuple[torch\u001b[38;5;241m.\u001b[39mTensor]:\n\u001b[1;32m--> 427\u001b[0m     self_outputs \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mself\u001b[49m\u001b[43m(\u001b[49m\n\u001b[0;32m    428\u001b[0m \u001b[43m        \u001b[49m\u001b[43mhidden_states\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    429\u001b[0m \u001b[43m        \u001b[49m\u001b[43mattention_mask\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    430\u001b[0m \u001b[43m        \u001b[49m\u001b[43mhead_mask\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    431\u001b[0m \u001b[43m        \u001b[49m\u001b[43mencoder_hidden_states\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    432\u001b[0m \u001b[43m        \u001b[49m\u001b[43mencoder_attention_mask\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    433\u001b[0m \u001b[43m        \u001b[49m\u001b[43mpast_key_value\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    434\u001b[0m \u001b[43m        \u001b[49m\u001b[43moutput_attentions\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    435\u001b[0m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    436\u001b[0m     attention_output \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39moutput(self_outputs[\u001b[38;5;241m0\u001b[39m], hidden_states)\n\u001b[0;32m    437\u001b[0m     outputs \u001b[38;5;241m=\u001b[39m (attention_output,) \u001b[38;5;241m+\u001b[39m self_outputs[\u001b[38;5;241m1\u001b[39m:]  \u001b[38;5;66;03m# add attentions if we output them\u001b[39;00m\n",
      "File \u001b[1;32mc:\\gitProjects\\yeni\\.venv\\lib\\site-packages\\torch\\nn\\modules\\module.py:1553\u001b[0m, in \u001b[0;36mModule._wrapped_call_impl\u001b[1;34m(self, *args, **kwargs)\u001b[0m\n\u001b[0;32m   1551\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_compiled_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)  \u001b[38;5;66;03m# type: ignore[misc]\u001b[39;00m\n\u001b[0;32m   1552\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m-> 1553\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n",
      "File \u001b[1;32mc:\\gitProjects\\yeni\\.venv\\lib\\site-packages\\torch\\nn\\modules\\module.py:1562\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[1;34m(self, *args, **kwargs)\u001b[0m\n\u001b[0;32m   1557\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[0;32m   1558\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[0;32m   1559\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[0;32m   1560\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[0;32m   1561\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[1;32m-> 1562\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m forward_call(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n\u001b[0;32m   1564\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m   1565\u001b[0m     result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n",
      "File \u001b[1;32mc:\\gitProjects\\yeni\\.venv\\lib\\site-packages\\transformers\\models\\bert\\modeling_bert.py:308\u001b[0m, in \u001b[0;36mBertSelfAttention.forward\u001b[1;34m(self, hidden_states, attention_mask, head_mask, encoder_hidden_states, encoder_attention_mask, past_key_value, output_attentions)\u001b[0m\n\u001b[0;32m    306\u001b[0m     value_layer \u001b[38;5;241m=\u001b[39m torch\u001b[38;5;241m.\u001b[39mcat([past_key_value[\u001b[38;5;241m1\u001b[39m], value_layer], dim\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m2\u001b[39m)\n\u001b[0;32m    307\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m--> 308\u001b[0m     key_layer \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mtranspose_for_scores(\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mkey\u001b[49m\u001b[43m(\u001b[49m\u001b[43mhidden_states\u001b[49m\u001b[43m)\u001b[49m)\n\u001b[0;32m    309\u001b[0m     value_layer \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mtranspose_for_scores(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mvalue(hidden_states))\n\u001b[0;32m    311\u001b[0m query_layer \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mtranspose_for_scores(mixed_query_layer)\n",
      "File \u001b[1;32mc:\\gitProjects\\yeni\\.venv\\lib\\site-packages\\torch\\nn\\modules\\module.py:1553\u001b[0m, in \u001b[0;36mModule._wrapped_call_impl\u001b[1;34m(self, *args, **kwargs)\u001b[0m\n\u001b[0;32m   1551\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_compiled_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)  \u001b[38;5;66;03m# type: ignore[misc]\u001b[39;00m\n\u001b[0;32m   1552\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m-> 1553\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n",
      "File \u001b[1;32mc:\\gitProjects\\yeni\\.venv\\lib\\site-packages\\torch\\nn\\modules\\module.py:1562\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[1;34m(self, *args, **kwargs)\u001b[0m\n\u001b[0;32m   1557\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[0;32m   1558\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[0;32m   1559\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[0;32m   1560\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[0;32m   1561\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[1;32m-> 1562\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m forward_call(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n\u001b[0;32m   1564\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m   1565\u001b[0m     result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n",
      "File \u001b[1;32mc:\\gitProjects\\yeni\\.venv\\lib\\site-packages\\torch\\nn\\modules\\linear.py:117\u001b[0m, in \u001b[0;36mLinear.forward\u001b[1;34m(self, input)\u001b[0m\n\u001b[0;32m    116\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mforward\u001b[39m(\u001b[38;5;28mself\u001b[39m, \u001b[38;5;28minput\u001b[39m: Tensor) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m Tensor:\n\u001b[1;32m--> 117\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mF\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mlinear\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43minput\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mweight\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mbias\u001b[49m\u001b[43m)\u001b[49m\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "from pymongo import MongoClient\n",
    "from sentence_transformers import SentenceTransformer\n",
    "import numpy as np \n",
    "from sklearn.metrics.pairwise import cosine_similarity\n",
    "import spacy\n",
    "import pandas as pd\n",
    "\n",
    "# Model ve NLP yükleme\n",
    "model = SentenceTransformer('sentence-transformers/paraphrase-MiniLM-L6-v2')\n",
    "nlp = spacy.load('en_core_web_sm')\n",
    "\n",
    "# MongoDB bağlantısı\n",
    "client = MongoClient(\"mongodb://localhost:27017/\")  \n",
    "db = client['EgitimDatabase']  \n",
    "collection = db['test']\n",
    "\n",
    "# Subheading'leri çıkarma\n",
    "def extract_subheadings(text):\n",
    "    doc = nlp(text)\n",
    "    sentences = [sent.text for sent in doc.sents]\n",
    "    return sentences\n",
    "\n",
    "# Metinleri gömme (embedding) işlemi\n",
    "def embed_text(text):\n",
    "    return model.encode(text)\n",
    "\n",
    "# En iyi subheading'leri seçme\n",
    "def get_top_subheadings(title, keywords, text, top_n=5):\n",
    "    title_embedding = embed_text(title)\n",
    "    keywords_embedding = embed_text(' '.join(keywords))\n",
    "    subheadings = extract_subheadings(text)\n",
    "    subheadings_embeddings = [embed_text(sub) for sub in subheadings]\n",
    "    \n",
    "    scores = [cosine_similarity([title_embedding], [embedding])[0][0] + \n",
    "              cosine_similarity([keywords_embedding], [embedding])[0][0]\n",
    "              for embedding in subheadings_embeddings]\n",
    "    \n",
    "    top_indices = np.argsort(scores)[-top_n:]\n",
    "    top_subheadings = [subheadings[i] for i in top_indices]\n",
    "\n",
    "    return top_subheadings\n",
    "\n",
    "# Verileri işleme ve CSV'ye yazma\n",
    "def process_documents(batch_size=1000, top_n_subheadings=5):\n",
    "    data = []\n",
    "    total_documents = 10000\n",
    "    for skip in range(0, total_documents, batch_size):\n",
    "        documents = collection.find({}, {'keywords': 1, 'title': 1, 'text': 1}).skip(skip).limit(batch_size)\n",
    "        for doc in documents:\n",
    "            title = doc['title']\n",
    "            keywords = doc['keywords']\n",
    "            text = doc['text']\n",
    "            \n",
    "            top_subheadings = get_top_subheadings(title, keywords, text)\n",
    "            \n",
    "            # Verileri listeye ekleyin\n",
    "            data.append({\n",
    "                'Title': title,\n",
    "                'Keywords': ', '.join(keywords),\n",
    "                'Subheadings': '; '.join(top_subheadings)\n",
    "            })\n",
    "        \n",
    "        # Her batch sonrası CSV dosyasına kaydetme\n",
    "        df = pd.DataFrame(data)\n",
    "        df.to_csv('subheadings.csv', index=False, mode='a', header=not bool(skip))\n",
    "        print(f\"Batch {skip // batch_size + 1} işlenip kaydedildi.\")\n",
    "    \n",
    "    print(\"Subheadings başarıyla 'subheadings.csv' dosyasına kaydedildi.\")\n",
    "\n",
    "# CSV'ye yazma işlemini başlatma\n",
    "process_documents()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "def generate_ngrams(text, n):\n",
    "    tokens = word_tokenize(text) #[\"this\",\"is\",\"a\",\"test\"]\n",
    "    n_grams = ngrams(tokens, n) #kelime gruplarına böler\n",
    "    return [' '.join(gram) for gram in n_grams]\n",
    "\n",
    "def get_important_ngrams(text, n, top_n=5):\n",
    "    n_grams = generate_ngrams(text, n)\n",
    "    ngram_freq = Counter(n_grams)\n",
    "    most_common_ngrams = [ngram for ngram, _ in ngram_freq.most_common(top_n)]\n",
    "    return most_common_ngrams\n",
    "\n",
    "def get_similar_bigrams(text, n=2, similarity_threshold=0.8):\n",
    "    # Bigramları oluştur\n",
    "    bigrams = generate_ngrams(text, n)\n",
    "    \n",
    "    # Bigramları embedding vektörlerine dönüştür\n",
    "    bigram_embeddings = [embed_text(bigram) for bigram in bigrams]\n",
    "    \n",
    "    # Benzer bigramları listelemek için boş bir liste oluştur\n",
    "    similar_bigrams = []\n",
    "    \n",
    "    # Bigramlar arasındaki benzerlikleri kontrol et\n",
    "    for i in range(len(bigrams)):\n",
    "        for j in range(i + 1, len(bigrams)):\n",
    "            similarity = cosine_similarity([bigram_embeddings[i]], [bigram_embeddings[j]])[0][0]\n",
    "            if similarity >= similarity_threshold:\n",
    "                similar_bigrams.append((bigrams[i], bigrams[j], similarity))\n",
    "    \n",
    "    return similar_bigrams\n",
    "\n",
    "similar_bigrams = get_similar_bigrams(text, n=2, similarity_threshold=0.8)\n",
    "# Benzer bigramları yazdır\n",
    "for bigram1, bigram2, similarity in similar_bigrams:\n",
    "    print(f\"Bigram 1: {bigram1}, Bigram 2: {bigram2}, Similarity: {similarity:.2f}\")\n",
    "\n",
    "def generate_subheadings(text, num_headings=5):\n",
    "    # Metni spaCy ile işleyin\n",
    "    doc = nlp(text)\n",
    "    \n",
    "    # Biagram ve triagramları oluşturun\n",
    "    bigrams = get_important_ngrams(text, 2, top_n=num_headings)\n",
    "    trigrams = get_important_ngrams(text, 3, top_n=num_headings)\n",
    "    \n",
    "    # Subheadings'leri birleştirin\n",
    "    headings = list(set(bigrams + trigrams))\n",
    "    \n",
    "    return headings"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": ".venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}