{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from rdkit import Chem\n",
    "from rdkit.Chem import AllChem\n",
    "import numpy as np\n",
    "from scipy import stats\n",
    "import pandas as pd\n",
    "from tqdm import tqdm\n",
    "from constants import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "DATABASE_PATH_DICT = {\n",
    "    r\"ALL\": f\"{DATA_PATH}/nps_intersected\",\n",
    "    r\"ALMANAC-skin\": f\"{DATA_PATH}/nps_intersected/ALMANAC/skin\",\n",
    "    r\"ALMANAC-ovary\": f\"{DATA_PATH}/nps_intersected/ALMANAC/ovary\",\n",
    "    r\"ALMANAC-lung\": f\"{DATA_PATH}/nps_intersected/ALMANAC/lung\",\n",
    "    r\"ALMANAC-breast\": f\"{DATA_PATH}/nps_intersected/ALMANAC/breast\",\n",
    "    r\"FRIEDMAN\": f\"{DATA_PATH}/nps_intersected/FRIEDMAN\",\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "def remove_useless_descriptor(df, threshold):\n",
    "    des1 = len(df.columns)\n",
    "\n",
    "    h = df.columns.tolist()\n",
    "    df = df.to_numpy().astype(np.float)\n",
    "    df = np.array(df)\n",
    "\n",
    "    STDEV = np.std(df, axis=0)\n",
    "    idx = [idx for idx, val in enumerate(STDEV) if val > threshold]\n",
    "    df2 = df[:, idx]\n",
    "    hx = np.array(h)[idx]\n",
    "\n",
    "    df = pd.DataFrame(df2, columns=[hx])\n",
    "\n",
    "    des2 = len(df.columns)\n",
    "\n",
    "    print(\"from Remove useless descriptor\")\n",
    "    print(\n",
    "        \"The initial set of \"\n",
    "        + str(des1)\n",
    "        + \" descriptors has been reduced to \"\n",
    "        + str(des2)\n",
    "        + \" descriptors.\"\n",
    "    )\n",
    "\n",
    "    return df, des1, des2\n",
    "\n",
    "\n",
    "def correlation(df, threshold):\n",
    "    des3 = len(df.columns)\n",
    "    corr = stats.pearsonr\n",
    "    col_corr = set()  # Set of all the names of deleted columns\n",
    "    corr_matrix = df.corr()\n",
    "    for i in range(len(corr_matrix.columns)):\n",
    "        for j in range(i):\n",
    "            if corr_matrix.iloc[i, j] >= threshold:\n",
    "                colname = corr_matrix.columns[i]  # getting the name of column\n",
    "                col_corr.add(colname)\n",
    "                if colname in df.columns:\n",
    "                    del df[colname]  # deleting the column from the dataset\n",
    "    des4 = len(df.columns)\n",
    "\n",
    "    print(\"from Remove correlation\")\n",
    "    print(\n",
    "        \"The initial set of \"\n",
    "        + str(des3)\n",
    "        + \" descriptors\"\n",
    "        + \" has been reduced to \"\n",
    "        + str(des4)\n",
    "        + \" descriptors.\"\n",
    "    )\n",
    "\n",
    "    return df, des3, des4\n",
    "\n",
    "def get_ecfp6_map(smiles, nBits=2048, r=3):\n",
    "    print(smiles)\n",
    "    smiles_unique = pd.Series(np.unique(smiles))\n",
    "    mol = smiles_unique.apply(Chem.MolFromSmiles)\n",
    "    fp1 = mol.apply(lambda m: AllChem.GetMorganFingerprintAsBitVect(m, r, nBits=nBits))\n",
    "    map_smiles_to_fp = dict(zip(smiles_unique.tolist(), fp1.tolist()))\n",
    "    return map_smiles_to_fp\n",
    "\n",
    "def get_ecfp6_list(smileslist, r=3, nBits=2048):\n",
    "    # ret = np.zeros((len(smileslist), nBits))\n",
    "    # for idx, smiles in tqdm(enumerate(smileslist)):\n",
    "    #     ret[idx] = get_ecfp6(smiles, nBits, r)\n",
    "    # return ret\n",
    "    print(\"Getting map...\")\n",
    "    map_smiles_to_fp = get_ecfp6_map(smileslist)\n",
    "    print(\"Getting stack of ecfp6...\")\n",
    "    ret = np.stack(pd.Series(smileslist).map(map_smiles_to_fp).tolist())\n",
    "    print(\"Done\")\n",
    "    return ret\n",
    "\n",
    "\n",
    "def clean_and_save(database):\n",
    "    print(\"~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\")\n",
    "    print(f\"Cleaning and Saving {database}\")\n",
    "    print(\"~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\")\n",
    "    # load dataset\n",
    "    print(\"Loading dataset...\")\n",
    "    database_path = DATABASE_PATH_DICT[database]\n",
    "    ID = np.load(f\"{database_path}/idx_row.npy\", allow_pickle=True)\n",
    "    dataframe = pd.read_pickle(f\"{EXPORT_DATA_PATH}/data_drugcomb.pkl\")\n",
    "    smiles_col = dataframe[\"molecule_structures_col\"].loc[ID].values\n",
    "    smiles_row = dataframe[\"molecule_structures_row\"].loc[ID].values\n",
    "\n",
    "    smiles = np.concatenate((smiles_row, smiles_col))\n",
    "    print(smiles.shape)\n",
    "    print(\"Done load dataset!\")\n",
    "\n",
    "    # get ecfp6\n",
    "    print(\"Converting dataset to ecfp6...\")\n",
    "    data_col = get_ecfp6_list(smiles_col)\n",
    "    data_row = get_ecfp6_list(smiles_row)\n",
    "    print(\"Done converting dataset to ecfp6!\")\n",
    "\n",
    "    print(\"Saving..\")\n",
    "    np.save(f\"{database_path}/deepsyn_drug_row.npy\", data_row)\n",
    "    np.save(f\"{database_path}/deepsyn_drug_col.npy\", data_col)\n",
    "    print(\"Saved!\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n",
      "Cleaning and Saving ALL\n",
      "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n",
      "Loading dataset...\n",
      "(239200,)\n",
      "Done load dataset!\n",
      "Converting dataset to ecfp6...\n",
      "Getting map...\n",
      "['Cn1nnc2c(C(N)=O)ncn2c1=O' 'Cn1nnc2c(C(N)=O)ncn2c1=O'\n",
      " 'Cn1nnc2c(C(N)=O)ncn2c1=O' ...\n",
      " 'N#CC[C@H](C1CCCC1)n1cc(-c2ncnc3[nH]ccc23)cn1'\n",
      " 'N#CC[C@H](C1CCCC1)n1cc(-c2ncnc3[nH]ccc23)cn1'\n",
      " 'N#CC[C@H](C1CCCC1)n1cc(-c2ncnc3[nH]ccc23)cn1']\n",
      "Getting stack of ecfp6...\n",
      "Done\n",
      "Getting map...\n",
      "['CN1CCc2cc3c(cc2[C@H]1[C@@H]1OC(=O)c2c1ccc1c2OCO1)OCO3'\n",
      " 'CCCN1CCO[C@@H]2c3cc(O)ccc3OC[C@H]21.Cl'\n",
      " 'CC(=O)C1=C(O)[C@]2(C)C(=CC1=O)Oc1c(C(C)=O)c(O)c(C)c(O)c12' ...\n",
      " 'Cc1ccccc1C(=O)Nc1ccc(C(=O)N2CCCC(O)c3cc(Cl)ccc32)c(C)c1'\n",
      " 'Cc1cccc(Nc2ccncc2S(=O)(=O)NC(=O)NC(C)C)c1'\n",
      " 'CCOC(=O)N1c2ccc(C(F)(F)F)cc2[C@@H](N(Cc2cc(C(F)(F)F)cc(C(F)(F)F)c2)C(=O)OC)C[C@H]1CC']\n",
      "Getting stack of ecfp6...\n",
      "Done\n",
      "Done converting dataset to ecfp6!\n",
      "Saving..\n",
      "Saved!\n",
      "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n",
      "Cleaning and Saving ALMANAC-skin\n",
      "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n",
      "Loading dataset...\n",
      "(24170,)\n",
      "Done load dataset!\n",
      "Converting dataset to ecfp6...\n",
      "Getting map...\n",
      "['Cn1nnc2c(C(N)=O)ncn2c1=O' 'Cn1nnc2c(C(N)=O)ncn2c1=O'\n",
      " 'Cn1nnc2c(C(N)=O)ncn2c1=O' ...\n",
      " 'C[C@]12CC[C@H]3[C@@H](CC=C4C[C@@H](O)CC[C@@]43C)[C@@H]1CC=C2c1cccnc1'\n",
      " 'C[C@]12CC[C@H]3[C@@H](CC=C4C[C@@H](O)CC[C@@]43C)[C@@H]1CC=C2c1cccnc1'\n",
      " 'C[C@]12CC[C@H]3[C@@H](CC=C4C[C@@H](O)CC[C@@]43C)[C@@H]1CC=C2c1cccnc1']\n",
      "Getting stack of ecfp6...\n",
      "Done\n",
      "Getting map...\n",
      "['COc1cccc2c1C(=O)c1c(O)c3c(c(O)c1C2=O)C[C@@](O)(C(C)=O)C[C@@H]3O[C@H]1C[C@H](N)[C@H](O)[C@H](C)O1.Cl'\n",
      " 'O=c1[nH]cc(F)c(=O)[nH]1' 'Nc1nc2[nH]cnc2c(=S)[nH]1' ...\n",
      " 'CCCS(=O)(=O)Nc1ccc(F)c(C(=O)c2c[nH]c3ncc(-c4ccc(Cl)cc4)cc23)c1F'\n",
      " 'CS(=O)(=O)c1ccc(C(=O)Nc2ccc(Cl)c(-c3ccccn3)c2)c(Cl)c1'\n",
      " 'O=C(c1ccc(OCCN2CCCCC2)cc1)c1c(-c2ccc(O)cc2)sc2cc(O)ccc12']\n",
      "Getting stack of ecfp6...\n",
      "Done\n",
      "Done converting dataset to ecfp6!\n",
      "Saving..\n",
      "Saved!\n",
      "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n",
      "Cleaning and Saving ALMANAC-ovary\n",
      "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n",
      "Loading dataset...\n",
      "(26376,)\n",
      "Done load dataset!\n",
      "Converting dataset to ecfp6...\n",
      "Getting map...\n",
      "['Cn1nnc2c(C(N)=O)ncn2c1=O' 'Cn1nnc2c(C(N)=O)ncn2c1=O'\n",
      " 'Cn1nnc2c(C(N)=O)ncn2c1=O' ...\n",
      " 'C[C@]12CC[C@H]3[C@@H](CC=C4C[C@@H](O)CC[C@@]43C)[C@@H]1CC=C2c1cccnc1'\n",
      " 'C[C@]12CC[C@H]3[C@@H](CC=C4C[C@@H](O)CC[C@@]43C)[C@@H]1CC=C2c1cccnc1'\n",
      " 'C[C@]12CC[C@H]3[C@@H](CC=C4C[C@@H](O)CC[C@@]43C)[C@@H]1CC=C2c1cccnc1']\n",
      "Getting stack of ecfp6...\n",
      "Done\n",
      "Getting map...\n",
      "['COc1cccc2c1C(=O)c1c(O)c3c(c(O)c1C2=O)C[C@@](O)(C(C)=O)C[C@@H]3O[C@H]1C[C@H](N)[C@H](O)[C@H](C)O1.Cl'\n",
      " 'O=c1[nH]c(=O)n(C2CC(O)C(CO)O2)cc1F' 'O=c1[nH]cc(F)c(=O)[nH]1' ...\n",
      " 'CCCS(=O)(=O)Nc1ccc(F)c(C(=O)c2c[nH]c3ncc(-c4ccc(Cl)cc4)cc23)c1F'\n",
      " 'CS(=O)(=O)c1ccc(C(=O)Nc2ccc(Cl)c(-c3ccccn3)c2)c(Cl)c1'\n",
      " 'O=C(c1ccc(OCCN2CCCCC2)cc1)c1c(-c2ccc(O)cc2)sc2cc(O)ccc12']\n",
      "Getting stack of ecfp6...\n",
      "Done\n",
      "Done converting dataset to ecfp6!\n",
      "Saving..\n",
      "Saved!\n",
      "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n",
      "Cleaning and Saving ALMANAC-lung\n",
      "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n",
      "Loading dataset...\n",
      "(45630,)\n",
      "Done load dataset!\n",
      "Converting dataset to ecfp6...\n",
      "Getting map...\n",
      "['Cn1nnc2c(C(N)=O)ncn2c1=O' 'Cn1nnc2c(C(N)=O)ncn2c1=O'\n",
      " 'Cn1nnc2c(C(N)=O)ncn2c1=O' ...\n",
      " 'C[C@]12CC[C@H]3[C@@H](CC=C4C[C@@H](O)CC[C@@]43C)[C@@H]1CC=C2c1cccnc1'\n",
      " 'C[C@]12CC[C@H]3[C@@H](CC=C4C[C@@H](O)CC[C@@]43C)[C@@H]1CC=C2c1cccnc1'\n",
      " 'C[C@]12CC[C@H]3[C@@H](CC=C4C[C@@H](O)CC[C@@]43C)[C@@H]1CC=C2c1cccnc1']\n",
      "Getting stack of ecfp6...\n",
      "Done\n",
      "Getting map...\n",
      "['COc1cccc2c1C(=O)c1c(O)c3c(c(O)c1C2=O)C[C@@](O)(C(C)=O)C[C@@H]3O[C@H]1C[C@H](N)[C@H](O)[C@H](C)O1.Cl'\n",
      " 'O=c1[nH]c(=O)n(C2CC(O)C(CO)O2)cc1F' 'O=c1[nH]cc(F)c(=O)[nH]1' ...\n",
      " 'CCCS(=O)(=O)Nc1ccc(F)c(C(=O)c2c[nH]c3ncc(-c4ccc(Cl)cc4)cc23)c1F'\n",
      " 'CS(=O)(=O)c1ccc(C(=O)Nc2ccc(Cl)c(-c3ccccn3)c2)c(Cl)c1'\n",
      " 'O=C(c1ccc(OCCN2CCCCC2)cc1)c1c(-c2ccc(O)cc2)sc2cc(O)ccc12']\n",
      "Getting stack of ecfp6...\n",
      "Done\n",
      "Done converting dataset to ecfp6!\n",
      "Saving..\n",
      "Saved!\n",
      "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n",
      "Cleaning and Saving ALMANAC-breast\n",
      "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n",
      "Loading dataset...\n",
      "(26532,)\n",
      "Done load dataset!\n",
      "Converting dataset to ecfp6...\n",
      "Getting map...\n",
      "['Cn1nnc2c(C(N)=O)ncn2c1=O' 'Cn1nnc2c(C(N)=O)ncn2c1=O'\n",
      " 'Cn1nnc2c(C(N)=O)ncn2c1=O' ...\n",
      " 'C[C@]12CC[C@H]3[C@@H](CC=C4C[C@@H](O)CC[C@@]43C)[C@@H]1CC=C2c1cccnc1'\n",
      " 'C[C@]12CC[C@H]3[C@@H](CC=C4C[C@@H](O)CC[C@@]43C)[C@@H]1CC=C2c1cccnc1'\n",
      " 'C[C@]12CC[C@H]3[C@@H](CC=C4C[C@@H](O)CC[C@@]43C)[C@@H]1CC=C2c1cccnc1']\n",
      "Getting stack of ecfp6...\n",
      "Done\n",
      "Getting map...\n",
      "['COc1cccc2c1C(=O)c1c(O)c3c(c(O)c1C2=O)C[C@@](O)(C(C)=O)C[C@@H]3O[C@H]1C[C@H](N)[C@H](O)[C@H](C)O1.Cl'\n",
      " 'O=c1[nH]c(=O)n(C2CC(O)C(CO)O2)cc1F' 'O=c1[nH]cc(F)c(=O)[nH]1' ...\n",
      " 'COc1cc2/c(=N/c3ccc(Br)cc3F)nc[nH]c2cc1OCC1CCN(C)CC1'\n",
      " 'CCCS(=O)(=O)Nc1ccc(F)c(C(=O)c2c[nH]c3ncc(-c4ccc(Cl)cc4)cc23)c1F'\n",
      " 'CS(=O)(=O)c1ccc(C(=O)Nc2ccc(Cl)c(-c3ccccn3)c2)c(Cl)c1']\n",
      "Getting stack of ecfp6...\n",
      "Done\n",
      "Done converting dataset to ecfp6!\n",
      "Saving..\n",
      "Saved!\n",
      "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n",
      "Cleaning and Saving FRIEDMAN\n",
      "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n",
      "Loading dataset...\n",
      "(61188,)\n",
      "Done load dataset!\n",
      "Converting dataset to ecfp6...\n",
      "Getting map...\n",
      "['Cn1nnc2c(C(N)=O)ncn2c1=O' 'Cn1nnc2c(C(N)=O)ncn2c1=O'\n",
      " 'Cn1nnc2c(C(N)=O)ncn2c1=O' ...\n",
      " 'COc1cc([C@@H]2c3cc4c(cc3[C@@H](O[C@@H]3O[C@@H]5CO[C@@H](C)O[C@H]5[C@H](O)[C@H]3O)[C@H]3COC(=O)[C@H]23)OCO4)cc(OC)c1O'\n",
      " 'COc1cc([C@@H]2c3cc4c(cc3[C@@H](O[C@@H]3O[C@@H]5CO[C@@H](C)O[C@H]5[C@H](O)[C@H]3O)[C@H]3COC(=O)[C@H]23)OCO4)cc(OC)c1O'\n",
      " 'COc1cc([C@@H]2c3cc4c(cc3[C@@H](O[C@@H]3O[C@@H]5CO[C@@H](C)O[C@H]5[C@H](O)[C@H]3O)[C@H]3COC(=O)[C@H]23)OCO4)cc(OC)c1O']\n",
      "Getting stack of ecfp6...\n",
      "Done\n",
      "Getting map...\n",
      "['COCC[n+]1c2c(n(Cc3cnccn3)c1C)C(=O)c1ccccc1C2=O.[Br-]'\n",
      " 'Cn1c(=O)n(-c2ccc(C(C)(C)C#N)cc2)c2c3cc(-c4cnc5ccccc5c4)ccc3ncc21'\n",
      " 'O=[N+]([O-])c1cnc(Sc2nnc(O)n2-c2ccc3c(c2)OCCO3)s1' ...\n",
      " 'N#CC[C@H](C1CCCC1)n1cc(-c2ncnc3[nH]ccc23)cn1'\n",
      " 'Cc1cc(Nc2cc(N3CCN(C)CC3)nc(Sc3ccc(NC(=O)C4CC4)cc3)n2)n[nH]1'\n",
      " 'CC[C@]1(O)C[C@H]2CN(CCc3c([nH]c4ccccc34)[C@@](C(=O)OC)(c3cc4c(cc3OC)N(C=O)[C@H]3[C@@](O)(C(=O)OC)[C@H](OC(C)=O)[C@]5(CC)C=CCN6CC[C@]43[C@@H]65)C2)C1']\n",
      "Getting stack of ecfp6...\n",
      "Done\n",
      "Done converting dataset to ecfp6!\n",
      "Saving..\n",
      "Saved!\n"
     ]
    }
   ],
   "source": [
    "for s in DATABASE_PATH_DICT.keys():\n",
    "    clean_and_save(s)"
   ]
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "1e699fdd3598ba0dbc1fc0137f52b6243f2eb0087639281a57a5a36e9da9a0f1"
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
