{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Local Effects"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "One of the more important aspects of random forest nodes, and by extension node clusters, is that they describe what we would call \"Local Effects\"\n",
    "\n",
    "While a conventional linear regression might describe a linear relationship between the behavior of a feature and a target that is true across the entire dataset, a node in a random forest may just as easily be a child of another node, and thus only trained on a small part of the dataset. Therefore a relationship that it describes between a feature and a target may be true across the entire dataset, or it may only be true conditionally on the predictions made by the parents of the node."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import scanpy as sc\n",
    "\n",
    "import sys\n",
    "# sys.path.append('/localscratch/bbrener1/rusty_forest_v3/src')\n",
    "sys.path.append('../src')\n",
    "import tree_reader as tr \n",
    "import lumberjack\n",
    "\n",
    "import pickle \n",
    "\n",
    "data_location = \"../data/aging_brain/\"\n",
    "\n",
    "young = pickle.load(open(data_location + \"aging_brain_young.pickle\",mode='rb'))\n",
    "old = pickle.load(open(data_location + \"aging_brain_old.pickle\",mode='rb'))\n",
    "\n",
    "forest = tr.Forest.load(data_location + 'full_clustering')\n",
    "forest.arguments"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# forest.reset_split_clusters()\n",
    "\n",
    "# forest.interpret_splits(\n",
    "#     depth=8,\n",
    "#     mode='additive_mean',\n",
    "#     metric='cosine',\n",
    "#     pca=100,\n",
    "#     relatives=True,\n",
    "#     k=50,\n",
    "#     resolution=2,\n",
    "# )\n",
    "\n",
    "# print(len(forest.split_clusters))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# forest.maximum_spanning_tree(mode='samples')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# forest.html_tree_summary(n=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# forest.backup(data_location + \"full_clustering\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# We now would like to see if there are any local associations that are dramatically different\n",
    "# from global ones, to the degree that it is impossible to recapture them using PCA-based analysis. \n",
    "\n",
    "# We will need to perform a PCA analysis first. \n",
    "\n",
    "from sklearn.decomposition import PCA\n",
    "\n",
    "model = PCA(n_components=25).fit(young.X)\n",
    "transformed = model.transform(young.X)\n",
    "recovered = model.inverse_transform(transformed)\n",
    "\n",
    "centered = young.X - np.mean(young.X,axis=0)\n",
    "null_squared_residual = np.power(centered,2)\n",
    "\n",
    "recovered_residual = young.X - recovered\n",
    "recovered_squared_residual = np.power(recovered_residual,2)\n",
    "\n",
    "pca_recovered_per_sample = np.sum(recovered_squared_residual,axis=1)\n",
    "pca_recovered_fraction_per_sample = np.sum(recovered_squared_residual,axis=1) / np.sum(null_squared_residual,axis=1)\n",
    "print(np.sum(null_squared_residual))\n",
    "print(np.sum(recovered_squared_residual))\n",
    "\n",
    "print(f\"Remaining variance:{(np.sum(recovered_squared_residual) / np.sum(null_squared_residual))}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "for i,pc in enumerate(transformed.T):\n",
    "    plt.figure()\n",
    "    plt.title(i)\n",
    "    ab_max = np.max(np.abs(pc))\n",
    "    plt.scatter(*forest.tsne_coordinates.T,c=pc,s=3,alpha=.4,cmap='bwr',vmin=-ab_max,vmax=ab_max)\n",
    "    plt.colorbar()\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Now we will look for features that have an especially large discrepancy in the local \n",
    "# correlation compared to the global correlation for each factor. \n",
    "\n",
    "for factor in forest.split_clusters:\n",
    "    print(\"=====================================\")\n",
    "    print(factor.name())\n",
    "    print(\"=====================================\")\n",
    "    fi_pairs = factor.most_local_correlations()\n",
    "    features = forest.output_features\n",
    "    f_names = [(features[i],features[j]) for (i,j) in fi_pairs]\n",
    "    local_correlations = factor.local_correlations()\n",
    "    global_correlations = forest.global_correlations()\n",
    "    discrepancy = [(local_correlations[i,j],global_correlations[i,j]) for (i,j) in fi_pairs]\n",
    "    print(f_names)\n",
    "    print(discrepancy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "interesting_pairs = []\n",
    "\n",
    "for factor in forest.split_clusters:\n",
    "    interesting_pairs.extend(factor.most_local_correlations(n=1))\n",
    "    \n",
    "uniques = list(set([y for x in interesting_pairs for y in x]))\n",
    "  \n",
    "factor_correlation_table = np.zeros((len(interesting_pairs),len(forest.split_clusters)))\n",
    "\n",
    "for i,factor in enumerate(forest.split_clusters):\n",
    "    local_correlations = factor.local_correlations(indices=uniques)\n",
    "    for j,(f1,f2) in enumerate(interesting_pairs):\n",
    "        f1_u = uniques.index(f1)\n",
    "        f2_u = uniques.index(f2)\n",
    "        factor_correlation_table[j,i] = local_correlations[f1_u,f2_u]\n",
    "\n",
    "plt.figure()\n",
    "plt.imshow(factor_correlation_table,interpolation='none',aspect='auto',cmap='bwr',vmin=-1,vmax=1)\n",
    "plt.colorbar()\n",
    "plt.show()\n",
    "\n",
    "from scipy.cluster.hierarchy import linkage,dendrogram\n",
    "\n",
    "factor_agglomeration = dendrogram(linkage(factor_correlation_table, metric='cosine', method='average'), no_plot=True)['leaves']\n",
    "\n",
    "plt.figure()\n",
    "plt.imshow(factor_correlation_table.T[9 factor_agglomeration].T,interpolation='none',aspect='auto',cmap='bwr',vmin=-1,vmax=1)\n",
    "plt.colorbar()\n",
    "plt.show()\n",
    "\n",
    "print([(x,y) for x,y in enumerate(interesting_pairs)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "forest.output_features[1639]\n",
    "\n",
    "print(forest.split_clusters[23].local_correlations(indices=[717,1639]))\n",
    "print(forest.split_clusters[20].local_correlations(indices=[717,1639]))\n",
    "\n",
    "# cluster 23, Rrares2 (717), Meg3 (1639)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from scipy.stats import linregress\n",
    "\n",
    "f1 = \"Tmem119\"\n",
    "f2 = \"Cd74\"\n",
    "\n",
    "f1_index = forest.truth_dictionary.feature_dictionary[f1]\n",
    "f2_index = forest.truth_dictionary.feature_dictionary[f2]\n",
    "\n",
    "f1_values = forest.output[:,f1_index]\n",
    "f2_values = forest.output[:,f2_index]\n",
    "\n",
    "slope,intercept,r_fit,_,_ = linregress(f1_values,f2_values)\n",
    "\n",
    "plt.figure()\n",
    "plt.title(f\"Linar Fit, {f1}, {f2}, Naive\")\n",
    "plt.scatter(f1_values,f2_values,s=3)\n",
    "plt.plot(np.arange(7), intercept + (np.arange(7) * slope),c='red',label=f\"Slope:{np.around(slope,3)},R2:{np.around(r_fit,3)}\")\n",
    "plt.legend()\n",
    "plt.xlabel(f\"{f1}\")\n",
    "plt.ylabel(f\"{f2}\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "from scipy.stats import linregress\n",
    "\n",
    "factor = forest.split_clusters[34]\n",
    "\n",
    "# for factor in forest.split_clusters[1:]:\n",
    "\n",
    "factor_mask = np.abs(factor.sister_scores() > .2)\n",
    "\n",
    "# if np.sum(factor_mask.astype(dtype=int)) < 2:\n",
    "#     continue\n",
    "\n",
    "plt.figure()\n",
    "plt.hist(factor.sister_scores(),bins=50)\n",
    "plt.show()\n",
    "\n",
    "f1 = \"Tmem119\"\n",
    "f2 = \"Cd74\"\n",
    "\n",
    "f1_index = forest.truth_dictionary.feature_dictionary[f1]\n",
    "f2_index = forest.truth_dictionary.feature_dictionary[f2]\n",
    "\n",
    "f1_values = forest.output[:,f1_index][factor_mask]\n",
    "f2_values = forest.output[:,f2_index][factor_mask]\n",
    "\n",
    "slope,intercept,r_fit,_,_ = linregress(f1_values,f2_values)\n",
    "\n",
    "plt.figure()\n",
    "plt.title(f\"Linar Fit, {f1}, {f2}, Factor {factor.name()}, Filtered\")\n",
    "plt.scatter(f1_values,f2_values,s=3)\n",
    "plt.plot(np.arange(7), intercept + (np.arange(7) * slope),c='red',label=f\"Slope:{np.around(slope,3)},R2:{np.around(r_fit,3)}\")\n",
    "plt.xlabel(f\"{f1}\")\n",
    "plt.ylabel(f\"{f2}\")\n",
    "plt.legend()\n",
    "plt.show()\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "f1 = \"Cdk1\"\n",
    "f2 = \"Actg1\"\n",
    "\n",
    "f1_index = forest.truth_dictionary.feature_dictionary[f1]\n",
    "f2_index = forest.truth_dictionary.feature_dictionary[f2]\n",
    "\n",
    "f1_values = forest.output[:,f1_index]\n",
    "f2_values = forest.output[:,f2_index]\n",
    "\n",
    "slope,intercept,r_fit,_,_ = linregress(f1_values,f2_values)\n",
    "\n",
    "plt.figure()\n",
    "plt.title(f\"Linar Fit, {f1}, {f2}, Naive\")\n",
    "plt.scatter(f1_values,f2_values,s=3)\n",
    "plt.plot(np.arange(7), intercept + (np.arange(7) * slope),c='red',label=f\"Slope:{np.around(slope,3)},R2:{np.around(r_fit,3)}\")\n",
    "plt.legend()\n",
    "plt.xlabel(f\"{f1}\")\n",
    "plt.ylabel(f\"{f2}\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "f1 = \"Tmem119\"\n",
    "f2 = \"Cd74\"\n",
    "\n",
    "f1_index = forest.truth_dictionary.feature_dictionary[f1]\n",
    "f2_index = forest.truth_dictionary.feature_dictionary[f2]\n",
    "\n",
    "for i,component in enumerate(model.components_):\n",
    "    print(f\"{i}: {f1}:{component[f1_index]},{f2}:{component[f2_index]}\")\n",
    "\n",
    "plt.figure()\n",
    "plt.title(f\"PC Weights for {f1} and {f2}\")\n",
    "plt.scatter(model.components_[:,f1_index],model.components_[:,f2_index])\n",
    "plt.plot([.2,-.2],np.array([-.2,.2])*.55,color='red',label=\"Slope of -.55\")\n",
    "plt.legend()\n",
    "plt.xlabel(f1)\n",
    "plt.ylabel(f2)\n",
    "plt.show()\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sc.tl.rank_genes_groups(young,grouby)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}

