<commit_before>{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from __init__ import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "feature_x_sample = pd.read_csv(\"../input/feature_x_sample.tsv\", sep=\"\\t\", index_col=0)\n",
    "\n",
    "comparison_x_sample = pd.read_csv(\n",
    "    \"../output/comparison_x_sample.tsv\", sep=\"\\t\", index_col=0\n",
    ")\n",
    "\n",
    "set_to_features = kraft.gmt.read_many(kraft.path.list_(\"../input/set/\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "set_x_information = pd.DataFrame(\n",
    "    data={\n",
    "        \"Selected\": False,\n",
    "        \"N\": tuple(len(features) for features in set_to_features.values()),\n",
    "        \"n\": tuple(\n",
    "            sum(feature in feature_x_sample.index for feature in features)\n",
    "            for features in set_to_features.values()\n",
    "        ),\n",
    "    },\n",
    "    index=set_to_features,\n",
    ")\n",
    "\n",
    "set_x_information.loc[:, \".\"] = (\n",
    "    set_x_information.loc[:, \"n\"].to_numpy() / set_x_information.loc[:, \"N\"].to_numpy()\n",
    ")\n",
    "\n",
    "set_x_information"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "set_x_information.loc[\n",
    "    set(\n",
    "        kraft.series.get_extreme_labels(\n",
    "            set_x_information.loc[:, \"n\"],\n",
    "            \">\",\n",
    "            low_and_high=(None, SETTING[\"set_n\"]),\n",
    "            plot=False,\n",
    "        )\n",
    "    )\n",
    "    & set(\n",
    "        kraft.series.get_extreme_labels(\n",
    "            set_x_information.loc[:, \".\"],\n",
    "            \">\",\n",
    "            low_and_high=(None, SETTING[\"set_.\"]),\n",
    "            plot=False,\n",
    "        )\n",
    "    ),\n",
    "    \"Selected\",\n",
    "] = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "set_x_information.loc[SETTING[\"sets_to_peek\"], \"Selected\"] = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "set_x_information.loc[:, \"Selected\"].sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "set_x_information.to_csv(\"../output/set_x_information.tsv\", sep=\"\\t\")\n",
    "\n",
    "set_x_information"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "set_x_information.loc[SETTING[\"sets_to_peek\"], :]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for set_ in set_to_features:\n",
    "\n",
    "    if re.match(\n",
    "        r\"^{0}$|.*{1}{0}{1}.*|^{0}{1}.*|.*{1}{0}$\".format(\"CFS[0-9]*\", \"[\\s_]\"), set_,\n",
    "    ):\n",
    "\n",
    "        print(set_)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
<commit_msg>kraft.pandas<commit_after>{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from __init__ import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "feature_x_sample = pd.read_csv(\"../input/feature_x_sample.tsv\", sep=\"\\t\", index_col=0)\n",
    "\n",
    "comparison_x_sample = pd.read_csv(\n",
    "    \"../output/comparison_x_sample.tsv\", sep=\"\\t\", index_col=0\n",
    ")\n",
    "\n",
    "set_to_features = kraft.gmt.read_many(kraft.path.list_(\"../input/set/\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "set_x_information = pd.DataFrame(\n",
    "    data={\n",
    "        \"Selected\": False,\n",
    "        \"N\": tuple(len(features) for features in set_to_features.values()),\n",
    "        \"n\": tuple(\n",
    "            sum(feature in feature_x_sample.index for feature in features)\n",
    "            for features in set_to_features.values()\n",
    "        ),\n",
    "    },\n",
    "    index=set_to_features,\n",
    ")\n",
    "\n",
    "set_x_information.loc[:, \".\"] = (\n",
    "    set_x_information.loc[:, \"n\"].to_numpy() / set_x_information.loc[:, \"N\"].to_numpy()\n",
    ")\n",
    "\n",
    "set_x_information"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "set_x_information.loc[\n",
    "    set(\n",
    "        kraft.pandas.get_extreme_labels(\n",
    "            set_x_information.loc[:, \"n\"],\n",
    "            \">\",\n",
    "            low_and_high=(None, SETTING[\"set_n\"]),\n",
    "            plot=False,\n",
    "        )\n",
    "    )\n",
    "    & set(\n",
    "        kraft.pandas.get_extreme_labels(\n",
    "            set_x_information.loc[:, \".\"],\n",
    "            \">\",\n",
    "            low_and_high=(None, SETTING[\"set_.\"]),\n",
    "            plot=False,\n",
    "        )\n",
    "    ),\n",
    "    \"Selected\",\n",
    "] = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "set_x_information.loc[SETTING[\"sets_to_peek\"], \"Selected\"] = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "set_x_information.loc[:, \"Selected\"].sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "set_x_information.to_csv(\"../output/set_x_information.tsv\", sep=\"\\t\")\n",
    "\n",
    "set_x_information"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "set_x_information.loc[SETTING[\"sets_to_peek\"], :]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for set_ in set_to_features:\n",
    "\n",
    "    if re.match(\n",
    "        r\"^{0}$|.*{1}{0}{1}.*|^{0}{1}.*|.*{1}{0}$\".format(\"CFS[0-9]*\", \"[\\s_]\"), set_,\n",
    "    ):\n",
    "\n",
    "        print(set_)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}

{'fraction_non_alphanumeric': 0.25279762707294057, 'fraction_numerical': 0.0056626668464338685, 'mean_word_length': 2.1485568760611207, 'pattern_counts': {'":': 132, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1858661', 'n_tokens_mistral': 2972, 'n_tokens_neox': 2917, 'n_words': 394}