{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>input_text</th>\n",
       "      <th>target_text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>我要求的是法律上的澄清</td>\n",
       "      <td>我係要求……呢啲係好清楚嘅法律上嘅澄清呀</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>每晚由七點半，到十一點半</td>\n",
       "      <td>誒，由七點半就做到十一點半</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>梁頌恒議員，你是否要繼續發言</td>\n",
       "      <td>梁頌恆議員呢，係咪繼續係發言</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>可以怎樣稱呼我？我只知道整條街都稱我「大家姐」，因為我最大，年紀最大</td>\n",
       "      <td>可以點叫我呀？呢度成條街叫我大家姐，因為我最大，年紀最大吖嘛</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>至於他的答覆能否回應你剛才的提問，我並不能夠提出任何意見</td>\n",
       "      <td>噉呢，就對於佢能唔能夠達到你頭先提問嗰個嘅要求呢，我就唔能夠作出任何嘅意見</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35872</th>\n",
       "      <td>他曾在2006及2007年擔任暑期實習生</td>\n",
       "      <td>2006～2007學年寒暑假間亦試過將學校整大兼修容過</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35873</th>\n",
       "      <td>克里莫尼迪茲戰爭</td>\n",
       "      <td>克里米亞戰爭</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35874</th>\n",
       "      <td>產卵後親魚迴歸大海</td>\n",
       "      <td>海潮遇返失敗多年嘅生母</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35875</th>\n",
       "      <td>學校規模冠絕全馬。</td>\n",
       "      <td>學校嘅運動水平可謂全區之冠。</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35876</th>\n",
       "      <td>黃龍溪鎮也逐漸由繁忙的碼頭轉變為安靜的江邊場鎮。</td>\n",
       "      <td>而九龍寨城到海邊碼頭就慢慢變成市集。</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>35877 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                               input_text  \\\n",
       "0                             我要求的是法律上的澄清   \n",
       "1                            每晚由七點半，到十一點半   \n",
       "2                          梁頌恒議員，你是否要繼續發言   \n",
       "3      可以怎樣稱呼我？我只知道整條街都稱我「大家姐」，因為我最大，年紀最大   \n",
       "4            至於他的答覆能否回應你剛才的提問，我並不能夠提出任何意見   \n",
       "...                                   ...   \n",
       "35872                他曾在2006及2007年擔任暑期實習生   \n",
       "35873                            克里莫尼迪茲戰爭   \n",
       "35874                           產卵後親魚迴歸大海   \n",
       "35875                           學校規模冠絕全馬。   \n",
       "35876            黃龍溪鎮也逐漸由繁忙的碼頭轉變為安靜的江邊場鎮。   \n",
       "\n",
       "                                 target_text  \n",
       "0                       我係要求……呢啲係好清楚嘅法律上嘅澄清呀  \n",
       "1                              誒，由七點半就做到十一點半  \n",
       "2                             梁頌恆議員呢，係咪繼續係發言  \n",
       "3             可以點叫我呀？呢度成條街叫我大家姐，因為我最大，年紀最大吖嘛  \n",
       "4      噉呢，就對於佢能唔能夠達到你頭先提問嗰個嘅要求呢，我就唔能夠作出任何嘅意見  \n",
       "...                                      ...  \n",
       "35872            2006～2007學年寒暑假間亦試過將學校整大兼修容過  \n",
       "35873                                 克里米亞戰爭  \n",
       "35874                            海潮遇返失敗多年嘅生母  \n",
       "35875                         學校嘅運動水平可謂全區之冠。  \n",
       "35876                     而九龍寨城到海邊碼頭就慢慢變成市集。  \n",
       "\n",
       "[35877 rows x 2 columns]"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "\n",
    "df = pd.read_pickle(\"yue_zh_combined36k.pkl\")\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df.reset_index()  # make sure indexes pair with number of rows\n",
    "\n",
    "with open(\"train/mined_bitext.can\", \"w+\") as can_file, open(\"train/mined_bitext.man\", \"w+\") as man_file:\n",
    "    for index, row in df.iterrows():\n",
    "        man_file.write(row['input_text'] + \"\\n\")\n",
    "        can_file.write(row['target_text'] + \"\\n\")\n",
    "    man_file.flush()\n",
    "    can_file.flush()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.6"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}