{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# TODO: PCA partitions\n",
    "!pwd\n",
    "!ls ../../data/aging_brain/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import scanpy as sc\n",
    "\n",
    "import pickle \n",
    "\n",
    "data_location = \"../../data/aging_brain/\"\n",
    "\n",
    "young = pickle.load(open(data_location + \"aging_brain_young.pickle\",mode='rb'))\n",
    "old = pickle.load(open(data_location + \"aging_brain_old.pickle\",mode='rb'))\n",
    "\n",
    "# filtered = pickle.load(open(data_location + \"aging_brain_filtered.pickle\",mode='rb'))\n",
    "\n",
    "batch_encoding = np.loadtxt(data_location + 'aging_batch_encoding.tsv')\n",
    "batch_encoding = batch_encoding.astype(dtype=bool)\n",
    "\n",
    "young_mask = np.zeros(37069,dtype=bool)\n",
    "old_mask = np.zeros(37069,dtype=bool)\n",
    "\n",
    "young_mask[:young.shape[0]] = True\n",
    "old_mask[young.shape[0]:] = True\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sc.pp.neighbors(filtered)\n",
    "sc.tl.umap(filtered)\n",
    "sc.tl.louvain(filtered)\n",
    "sc.pl.umap(filtered,color='louvain')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# We must find all ways of combining 8 mice into 2 partitions of 4. \n",
    "\n",
    "# First find the binary representation of all numbers up to 256.\n",
    "\n",
    "up = np.unpackbits(np.arange(256).astype(dtype='uint8')).reshape((256,8))\n",
    "\n",
    "# Find out how many digits are 1\n",
    "\n",
    "sums = np.sum(up,axis=1)\n",
    "\n",
    "# Then select all representations where only 4 positions are set to 1.\n",
    "\n",
    "partitions = up[:128][sums[:128] == 4]\n",
    "\n",
    "partitions.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Now we must select features that are reasonably predictable within the dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Selecting Predictable Features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Setting context\n",
      "Input:(16028, 2000)\n",
      "Output:(16028, 2000)\n",
      "Generating trees\n",
      "Running /home/boris/battle/rf_5/rusty_axe/bin/rf_5\n",
      "Command: /home/boris/battle/rf_5/rusty_axe/bin/rf_5 -ic /tmp/tmplaubk4ks/input.counts -oc /tmp/tmplaubk4ks/output.counts -o /tmp/tmplaubk4ks/tmp -auto -ifh /tmp/tmplaubk4ks/tmp.ifh -ofh /tmp/tmplaubk4ks/tmp.ofh -p 6 -trees 50 -ifs 500 -ofs 500 -ss 300 -depth 9 -leaves 50 -sfr 0 -norm l2 -dispersion_mode ssme -standardize true -reduction 3 -reduce_input true -reduce_output false\n",
      "Read parameters\n",
      "\n",
      "0 0 1.8931357860565186 0 0 0 2.507997751235962 0 0 0 0\n",
      "100\n",
      "\n",
      "0 0 0 0 0 0 0 0 0 0 200\n",
      "300\n",
      "\n",
      "0 0 0 0 0 0 0 0 0 0 400\n",
      "500\n",
      "\n",
      "0 0 0 0 0 0 0 0 0 0 600\n",
      "700\n",
      "\n",
      "0 0 0 0 2.0017569065093994 0 0 0 0 0 800\n",
      "900\n",
      "\n",
      "0 0 0 0 0 0 0 1.5864790678024292 0 0 1000\n",
      "1100\n",
      "\n",
      "0 0 0 0 0 0 0 3.541308879852295 0 0 1200\n",
      "1300\n",
      "\n",
      "0 0 0 0 0 0 1.5142695903778076 0 0 0 1400\n",
      "1500\n",
      "\n",
      "0 0 0 0 0 0 0 0 0 0 1600\n",
      "1700\n",
      "\n",
      "0 0 0 0 0 0 0 0 0 0 1800\n",
      "1900\n",
      "\n",
      "0 0 0 0 0 0 0 0 0 0 2000\n",
      "2100\n",
      "\n",
      "0 0 0 0 1.4800764322280884 0 0 1.4800764322280884 0 0 2200\n",
      "2300\n",
      "\n",
      "0 0 0 0 1.5150998830795288 0 0 0 0 0 2400\n",
      "2500\n",
      "\n",
      "0 0 0 0 0 0 0 3.6586053371429443 0 0 2600\n",
      "2700\n",
      "\n",
      "0 0 0 0 0 0 0 2.3695943355560303 0 1.442442536354065 2800\n",
      "2900\n",
      "\n",
      "0 0 0 0 0 0 0 0 0 0 3000\n",
      "3100\n",
      "\n",
      "0 0 0 0 0 0 3.000828981399536 0 0 0 3200\n",
      "3300\n",
      "\n",
      "0 0 0 0 0 0 0 0 0 0 3400\n",
      "3500\n",
      "\n",
      "0 0 0 0 0 0 0 0 0 0 3600\n",
      "3700\n",
      "\n",
      "0 0 0 0 0 0 0 1.862636685371399 0 0 3800\n",
      "3900\n",
      "\n",
      "0 0 0 0 2.229140281677246 0 0 0 0 0 4000\n",
      "4100\n",
      "\n",
      "0 0 0 0 0 0 0 0 0 0 4200\n",
      "4300\n",
      "\n",
      "0 0 0 0 0 0 0 1.5815516710281372 0 0 4400\n",
      "4500\n",
      "\n",
      "0 0 0 0 0 0 0 0 0 0 4600\n",
      "4700\n",
      "\n",
      "0 1.214314579963684 0 0 0 0 0 0 0 0 4800\n",
      "4900\n",
      "\n",
      "0 0 0 0 0 0 3.1836531162261963 0 0 0 5000\n",
      "5100\n",
      "\n",
      "0 0 0 0 0 0 0 1.5914435386657715 0 0 5200\n",
      "5300\n",
      "\n",
      "0 0 0 0 0 0 0 0 0 0 5400\n",
      "5500\n",
      "\n",
      "0 0 0 0 0 0 0 0 0 0 5600\n",
      "5700\n",
      "\n",
      "0 0 0 0 0 0 0 0 0 0 5800\n",
      "5900\n",
      "\n",
      "0 0 0 0 0 0 0 1.6191076040267944 0 0 6000\n",
      "6100\n",
      "\n",
      "0 0 0 0 1.1573530435562134 0 0 0 0 0 6200\n",
      "6300\n",
      "\n",
      "0 0 0 0 0.5733009576797485 0 0 0 0 0 6400\n",
      "6500\n",
      "\n",
      "0 0 0 0 0 0 0 1.8346844911575317 0 1.8346844911575317 6600\n",
      "6700\n",
      "\n",
      "0 0 0 0 0 0 0 3.685375928878784 0 0 6800\n",
      "6900\n",
      "\n",
      "0 0 0 0 1.9447110891342163 0 0 3.6095566749572754 0 0 7000\n",
      "7100\n",
      "\n",
      "0 0 0 0 0 0 3.0982682704925537 0 0 0 7200\n",
      "7300\n",
      "\n",
      "0 0 0 0 0 0 0 0 0 0 7400\n",
      "7500\n",
      "\n",
      "0 0 0 0 0 0 0 0 0 0 7600\n",
      "7700\n",
      "\n",
      "0 0 0 0 0 0 0 0 0 0 7800\n",
      "7900\n",
      "\n",
      "0 0 0 0 0 0 0 0 0 0 8000\n",
      "8100\n",
      "\n",
      "0 0 0 0 0 0 0 0 0 0 8200\n",
      "8300\n",
      "\n",
      "0 0 0 0 0 0 0 1.0990124940872192 0 0 8400\n",
      "8500\n",
      "\n",
      "0 0 0 0 1.7238681316375732 0 1.7238681316375732 3.1793389320373535 0 0 8600\n",
      "8700\n",
      "\n",
      "0 1.8960157632827759 0 1.8960157632827759 0 0 0 3.5540859699249268 0 0 8800\n",
      "8900\n",
      "\n",
      "0 0 0 0 0 0 0 0 0 1.6033854484558105 9000\n",
      "9100\n",
      "\n",
      "0 0 0 0 0 0 0 0 0 0 9200\n",
      "9300\n",
      "\n",
      "0 0 0 0 0 0 0 0 0 0 9400\n",
      "9500\n",
      "\n",
      "0 0 0 0 0 0 0 0 0 0 9600\n",
      "9700\n",
      "\n",
      "0 0 0 0 0 0 0 0 1.674572467803955 0 9800\n",
      "9900\n",
      "\n",
      "0 0 0 0 0 0 0 0 0 0 10000\n",
      "10100\n",
      "\n",
      "0 0 0 0 1.3285235166549683 0 0 0 0 0 10200\n",
      "10300\n",
      "\n",
      "1.8285188674926758 1.8285188674926758 0 0 0 0 0 0 0 0 10400\n",
      "10500\n",
      "\n",
      "0 0 0 0 0 0 0 1.8828281164169312 0 1.0498963594436646 10600\n",
      "10700\n",
      "\n",
      "0 0 0 0 1.0425875186920166 0 0 2.8638525009155273 0 1.0425875186920166 10800\n",
      "10900\n",
      "\n",
      "0 0 0 0 0 0 0 0 0 0 11000\n",
      "11100\n",
      "\n",
      "0 0 0 0 0 0 0 0 0 0 11200\n",
      "11300\n",
      "\n",
      "0 0 0 0 0 0 0 0 0 0 11400\n",
      "11500\n",
      "\n",
      "0 0 0 0 0 0 0 0 0 0 11600\n",
      "11700\n",
      "\n",
      "0 0 0 0 0 0 0 0 0 0.8006526231765747 11800\n",
      "11900\n",
      "\n",
      "0 0 0 0 0 0 0 0 0 0 12000\n",
      "12100\n",
      "\n",
      "0 0 0 1.996789574623108 3.0005266666412354 0 0 0 0 1.996789574623108 12200\n",
      "12300\n",
      "\n",
      "0 0 0 0 0 0 0 1.679168939590454 0 0 12400\n",
      "12500\n",
      "\n",
      "0 0 0 0 0 0 0 0 0 0 12600\n",
      "12700\n",
      "\n",
      "0 1.8413492441177368 0 0 0 0 0 2.8282079696655273 0 0 12800\n",
      "12900\n",
      "\n",
      "0 0 0 0 0 0 0 2.3510587215423584 0 0 13000\n",
      "13100\n",
      "\n",
      "0 0 0 1.3673235177993774 0 0 0 1.3673235177993774 0 0 13200\n",
      "13300\n",
      "\n",
      "0 0 0 0 0 0 2.957416296005249 0 0 0 13400\n",
      "13500\n",
      "\n",
      "0 0 0 0 0 0 0 0 0 0 13600\n",
      "13700\n",
      "\n",
      "0 0 0 0 0 0 0 0 0 0 13800\n",
      "13900\n",
      "\n",
      "0 0 0 0 0 0 0 0 0 0 14000\n",
      "14100\n",
      "\n",
      "0 0 0 0 0 0 0 0 0 0 14200\n",
      "14300\n",
      "\n",
      "0 0 0 0 0 0 0.7154713869094849 0 0 0 14400\n",
      "14500\n",
      "\n",
      "0 0 0 0 0 0 0 0 0 0 14600\n",
      "14700\n",
      "\n",
      "0 0 0 0 0 0 0 0 0 0 14800\n",
      "14900\n",
      "\n",
      "0 0 0 0 1.0648548603057861 0 1.0648548603057861 1.0648548603057861 0 0 15000\n",
      "15100\n",
      "\n",
      "2.0710577964782715 1.197271466255188 0 0 0 0 0 0 0 0 15200\n",
      "15300\n",
      "\n",
      "0 2.195094108581543 0 0 1.6075206995010376 0 0 4.341439247131348 0 0 15400\n",
      "15500\n",
      "\n",
      "0 0 0 0 0 0 0 2.571714401245117 0 0 15600\n",
      "15700\n",
      "\n",
      "0 0 0 2.1566731929779053 0 0 0 0 0 0 15800\n",
      "15900\n",
      "\n",
      "0.9007956981658936 1.683558464050293 0.9007956981658936 0 0 0 1.3668876886367798 0 0 0 16000\n",
      "===========\n",
      "16028,2000\n",
      "\n",
      "0 0 1.8931357860565186 0 0 0 2.507997751235962 0 0 0 0\n",
      "100\n",
      "\n",
      "0 0 0 0 0 0 0 0 0 0 200\n",
      "300\n",
      "\n",
      "0 0 0 0 0 0 0 0 0 0 400\n",
      "500\n",
      "\n",
      "0 0 0 0 0 0 0 0 0 0 600\n",
      "700\n",
      "\n",
      "0 0 0 0 2.0017569065093994 0 0 0 0 0 800\n",
      "900\n",
      "\n",
      "0 0 0 0 0 0 0 1.5864790678024292 0 0 1000\n",
      "1100\n",
      "\n",
      "0 0 0 0 0 0 0 3.541308879852295 0 0 1200\n",
      "1300\n",
      "\n",
      "0 0 0 0 0 0 1.5142695903778076 0 0 0 1400\n",
      "1500\n",
      "\n",
      "0 0 0 0 0 0 0 0 0 0 1600\n",
      "1700\n",
      "\n",
      "0 0 0 0 0 0 0 0 0 0 1800\n",
      "1900\n",
      "\n",
      "0 0 0 0 0 0 0 0 0 0 2000\n",
      "2100\n",
      "\n",
      "0 0 0 0 1.4800764322280884 0 0 1.4800764322280884 0 0 2200\n",
      "2300\n",
      "\n",
      "0 0 0 0 1.5150998830795288 0 0 0 0 0 2400\n",
      "2500\n",
      "\n",
      "0 0 0 0 0 0 0 3.6586053371429443 0 0 2600\n",
      "2700\n",
      "\n",
      "0 0 0 0 0 0 0 2.3695943355560303 0 1.442442536354065 2800\n",
      "2900\n",
      "\n",
      "0 0 0 0 0 0 0 0 0 0 3000\n",
      "3100\n",
      "\n",
      "0 0 0 0 0 0 3.000828981399536 0 0 0 3200\n",
      "3300\n",
      "\n",
      "0 0 0 0 0 0 0 0 0 0 3400\n",
      "3500\n",
      "\n",
      "0 0 0 0 0 0 0 0 0 0 3600\n",
      "3700\n",
      "\n",
      "0 0 0 0 0 0 0 1.862636685371399 0 0 3800\n",
      "3900\n",
      "\n",
      "0 0 0 0 2.229140281677246 0 0 0 0 0 4000\n",
      "4100\n",
      "\n",
      "0 0 0 0 0 0 0 0 0 0 4200\n",
      "4300\n",
      "\n",
      "0 0 0 0 0 0 0 1.5815516710281372 0 0 4400\n",
      "4500\n",
      "\n",
      "0 0 0 0 0 0 0 0 0 0 4600\n",
      "4700\n",
      "\n",
      "0 1.214314579963684 0 0 0 0 0 0 0 0 4800\n",
      "4900\n",
      "\n",
      "0 0 0 0 0 0 3.1836531162261963 0 0 0 5000\n",
      "5100\n",
      "\n",
      "0 0 0 0 0 0 0 1.5914435386657715 0 0 5200\n",
      "5300\n",
      "\n",
      "0 0 0 0 0 0 0 0 0 0 5400\n",
      "5500\n",
      "\n",
      "0 0 0 0 0 0 0 0 0 0 5600\n",
      "5700\n",
      "\n",
      "0 0 0 0 0 0 0 0 0 0 5800\n",
      "5900\n",
      "\n",
      "0 0 0 0 0 0 0 1.6191076040267944 0 0 6000\n",
      "6100\n",
      "\n",
      "0 0 0 0 1.1573530435562134 0 0 0 0 0 6200\n",
      "6300\n",
      "\n",
      "0 0 0 0 0.5733009576797485 0 0 0 0 0 6400\n",
      "6500\n",
      "\n",
      "0 0 0 0 0 0 0 1.8346844911575317 0 1.8346844911575317 6600\n",
      "6700\n",
      "\n",
      "0 0 0 0 0 0 0 3.685375928878784 0 0 6800\n",
      "6900\n",
      "\n",
      "0 0 0 0 1.9447110891342163 0 0 3.6095566749572754 0 0 7000\n",
      "7100\n",
      "\n",
      "0 0 0 0 0 0 3.0982682704925537 0 0 0 7200\n",
      "7300\n",
      "\n",
      "0 0 0 0 0 0 0 0 0 0 7400\n",
      "7500\n",
      "\n",
      "0 0 0 0 0 0 0 0 0 0 7600\n",
      "7700\n",
      "\n",
      "0 0 0 0 0 0 0 0 0 0 7800\n",
      "7900\n",
      "\n",
      "0 0 0 0 0 0 0 0 0 0 8000\n",
      "8100\n",
      "\n",
      "0 0 0 0 0 0 0 0 0 0 8200\n",
      "8300\n",
      "\n",
      "0 0 0 0 0 0 0 1.0990124940872192 0 0 8400\n",
      "8500\n",
      "\n",
      "0 0 0 0 1.7238681316375732 0 1.7238681316375732 3.1793389320373535 0 0 8600\n",
      "8700\n",
      "\n",
      "0 1.8960157632827759 0 1.8960157632827759 0 0 0 3.5540859699249268 0 0 8800\n",
      "8900\n",
      "\n",
      "0 0 0 0 0 0 0 0 0 1.6033854484558105 9000\n",
      "9100\n",
      "\n",
      "0 0 0 0 0 0 0 0 0 0 9200\n",
      "9300\n",
      "\n",
      "0 0 0 0 0 0 0 0 0 0 9400\n",
      "9500\n",
      "\n",
      "0 0 0 0 0 0 0 0 0 0 9600\n",
      "9700\n",
      "\n",
      "0 0 0 0 0 0 0 0 1.674572467803955 0 9800\n",
      "9900\n",
      "\n",
      "0 0 0 0 0 0 0 0 0 0 10000\n",
      "10100\n",
      "\n",
      "0 0 0 0 1.3285235166549683 0 0 0 0 0 10200\n",
      "10300\n",
      "\n",
      "1.8285188674926758 1.8285188674926758 0 0 0 0 0 0 0 0 10400\n",
      "10500\n",
      "\n",
      "0 0 0 0 0 0 0 1.8828281164169312 0 1.0498963594436646 10600\n",
      "10700\n",
      "\n",
      "0 0 0 0 1.0425875186920166 0 0 2.8638525009155273 0 1.0425875186920166 10800\n",
      "10900\n",
      "\n",
      "0 0 0 0 0 0 0 0 0 0 11000\n",
      "11100\n",
      "\n",
      "0 0 0 0 0 0 0 0 0 0 11200\n",
      "11300\n",
      "\n",
      "0 0 0 0 0 0 0 0 0 0 11400\n",
      "11500\n",
      "\n",
      "0 0 0 0 0 0 0 0 0 0 11600\n",
      "11700\n",
      "\n",
      "0 0 0 0 0 0 0 0 0 0.8006526231765747 11800\n",
      "11900\n",
      "\n",
      "0 0 0 0 0 0 0 0 0 0 12000\n",
      "12100\n",
      "\n",
      "0 0 0 1.996789574623108 3.0005266666412354 0 0 0 0 1.996789574623108 12200\n",
      "12300\n",
      "\n",
      "0 0 0 0 0 0 0 1.679168939590454 0 0 12400\n",
      "12500\n",
      "\n",
      "0 0 0 0 0 0 0 0 0 0 12600\n",
      "12700\n",
      "\n",
      "0 1.8413492441177368 0 0 0 0 0 2.8282079696655273 0 0 12800\n",
      "12900\n",
      "\n",
      "0 0 0 0 0 0 0 2.3510587215423584 0 0 13000\n",
      "13100\n",
      "\n",
      "0 0 0 1.3673235177993774 0 0 0 1.3673235177993774 0 0 13200\n",
      "13300\n",
      "\n",
      "0 0 0 0 0 0 2.957416296005249 0 0 0 13400\n",
      "13500\n",
      "\n",
      "0 0 0 0 0 0 0 0 0 0 13600\n",
      "13700\n",
      "\n",
      "0 0 0 0 0 0 0 0 0 0 13800\n",
      "13900\n",
      "\n",
      "0 0 0 0 0 0 0 0 0 0 14000\n",
      "14100\n",
      "\n",
      "0 0 0 0 0 0 0 0 0 0 14200\n",
      "14300\n",
      "\n",
      "0 0 0 0 0 0 0.7154713869094849 0 0 0 14400\n",
      "14500\n",
      "\n",
      "0 0 0 0 0 0 0 0 0 0 14600\n",
      "14700\n",
      "\n",
      "0 0 0 0 0 0 0 0 0 0 14800\n",
      "14900\n",
      "\n",
      "0 0 0 0 1.0648548603057861 0 1.0648548603057861 1.0648548603057861 0 0 15000\n",
      "15100\n",
      "\n",
      "2.0710577964782715 1.197271466255188 0 0 0 0 0 0 0 0 15200\n",
      "15300\n",
      "\n",
      "0 2.195094108581543 0 0 1.6075206995010376 0 0 4.341439247131348 0 0 15400\n",
      "15500\n",
      "\n",
      "0 0 0 0 0 0 0 2.571714401245117 0 0 15600\n",
      "15700\n",
      "\n",
      "0 0 0 2.1566731929779053 0 0 0 0 0 0 15800\n",
      "15900\n",
      "\n",
      "0.9007956981658936 1.683558464050293 0.9007956981658936 0 0 0 1.3668876886367798 0 0 0 16000\n",
      "===========\n",
      "16028,2000\n",
      "Reading header: /tmp/tmplaubk4ks/tmp.ifh\n",
      "Read 2000 lines\n",
      "Reading header: /tmp/tmplaubk4ks/tmp.ofh\n",
      "Read 2000 lines\n",
      "Computing tree 3\n",
      "Computing tree 1\n",
      "Computing tree 25\n",
      "Computing tree 6\n",
      "Computing tree 0\n",
      "Computing tree 12\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Computing tree 2\n",
      "Computing tree 7\n",
      "Computing tree 4\n",
      "Computing tree 18\n",
      "Computing tree 26\n",
      "Computing tree 13\n",
      "Computing tree 5\n",
      "Computing tree 19\n",
      "Computing tree 21\n",
      "Computing tree 8\n",
      "Computing tree 27\n",
      "Computing tree 14\n",
      "Computing tree 23\n",
      "Computing tree 9\n",
      "Computing tree 22\n",
      "Computing tree 20\n",
      "Computing tree 28\n",
      "Computing tree 15\n",
      "Computing tree 24\n",
      "Computing tree 10\n",
      "Computing tree 11\n",
      "Computing tree 16\n",
      "Computing tree 29\n",
      "Computing tree 37\n",
      "Computing tree 17\n",
      "Computing tree 31\n",
      "Computing tree 34\n",
      "Computing tree 43\n",
      "Computing tree 30\n",
      "Computing tree 38\n",
      "Computing tree 40\n",
      "Computing tree 44\n",
      "Computing tree 32\n",
      "Computing tree 35\n",
      "Computing tree 39\n",
      "Computing tree 36\n",
      "Computing tree 41\n",
      "Computing tree 45\n",
      "Computing tree 42\n",
      "Computing tree 33\n",
      "Computing tree 46\n",
      "Computing tree 48\n",
      "Computing tree 47\n",
      "Computing tree 49\n",
      "\n",
      "\n",
      "Rust observes: /home/boris/battle/rf_5/rusty_axe/bin/rf_5\n",
      "\n",
      "CHECK OUTPUT\n",
      "['tmp.tree_19.compact', 'tmp.tree_49.compact', 'tmp.ofh', 'tmp.tree_39.compact', 'tmp.tree_32.compact', 'tmp.tree_38.compact', 'tmp.tree_45.compact', 'tmp.tree_43.compact', 'tmp.tree_36.compact', 'tmp.tree_37.compact', 'tmp.tree_41.compact', 'tmp.tree_7.compact', 'tmp.tree_28.compact', 'tmp.tree_46.compact', 'tmp.tree_1.compact', 'tmp.ifh', 'tmp.tree_48.compact', 'tmp.tree_21.compact', 'output.counts', 'tmp.tree_47.compact', 'tmp.tree_42.compact', 'tmp.tree_24.compact', 'tmp.tree_18.compact', 'tmp.tree_33.compact', 'tmp.tree_20.compact', 'tmp.tree_29.compact', 'tmp.tree_16.compact', 'tmp.tree_22.compact', 'tmp.tree_15.compact', 'tmp.tree_9.compact', 'tmp.tree_40.compact', 'tmp.tree_11.compact', 'tmp.tree_12.compact', 'input.counts', 'tmp.tree_14.compact', 'tmp.tree_30.compact', 'tmp.tree_31.compact', 'tmp.tree_4.compact', 'tmp.tree_3.compact', 'tmp.tree_5.compact', 'tmp.tree_34.compact', 'tmp.tree_6.compact', 'tmp.tree_17.compact', 'tmp.tree_13.compact', 'tmp.tree_27.compact', 'tmp.tree_10.compact', 'tmp.tree_44.compact', 'tmp.tree_0.compact', 'tmp.tree_25.compact', 'tmp.tree_2.compact', 'tmp.tree_8.compact', 'tmp.tree_26.compact', 'tmp.tree_23.compact', 'tmp.tree_35.compact']\n",
      "Looking for clusters:/tmp/tmplaubk4ks/tmp.clusters\n",
      "Loading /tmp/tmplaubk4ks/tmp.tree_0.compact\n",
      "Loading /tmp/tmplaubk4ks/tmp.tree_1.compact\n",
      "Loading /tmp/tmplaubk4ks/tmp.tree_10.compact\n",
      "Loading /tmp/tmplaubk4ks/tmp.tree_11.compact\n",
      "Loading /tmp/tmplaubk4ks/tmp.tree_12.compact\n",
      "Loading /tmp/tmplaubk4ks/tmp.tree_13.compact\n",
      "Loading /tmp/tmplaubk4ks/tmp.tree_14.compact\n",
      "Loading /tmp/tmplaubk4ks/tmp.tree_15.compact\n",
      "Loading /tmp/tmplaubk4ks/tmp.tree_16.compact\n",
      "Loading /tmp/tmplaubk4ks/tmp.tree_17.compact\n",
      "Loading /tmp/tmplaubk4ks/tmp.tree_18.compact\n",
      "Loading /tmp/tmplaubk4ks/tmp.tree_19.compact\n",
      "Loading /tmp/tmplaubk4ks/tmp.tree_2.compact\n",
      "Loading /tmp/tmplaubk4ks/tmp.tree_20.compact\n",
      "Loading /tmp/tmplaubk4ks/tmp.tree_21.compact\n",
      "Loading /tmp/tmplaubk4ks/tmp.tree_22.compact\n",
      "Loading /tmp/tmplaubk4ks/tmp.tree_23.compact\n",
      "Loading /tmp/tmplaubk4ks/tmp.tree_24.compact\n",
      "Loading /tmp/tmplaubk4ks/tmp.tree_25.compact\n",
      "Loading /tmp/tmplaubk4ks/tmp.tree_26.compact\n",
      "Loading /tmp/tmplaubk4ks/tmp.tree_27.compact\n",
      "Loading /tmp/tmplaubk4ks/tmp.tree_28.compact\n",
      "Loading /tmp/tmplaubk4ks/tmp.tree_29.compact\n",
      "Loading /tmp/tmplaubk4ks/tmp.tree_3.compact\n",
      "Loading /tmp/tmplaubk4ks/tmp.tree_30.compact\n",
      "Loading /tmp/tmplaubk4ks/tmp.tree_31.compact\n",
      "Loading /tmp/tmplaubk4ks/tmp.tree_32.compact\n",
      "Loading /tmp/tmplaubk4ks/tmp.tree_33.compact\n",
      "Loading /tmp/tmplaubk4ks/tmp.tree_34.compact\n",
      "Loading /tmp/tmplaubk4ks/tmp.tree_35.compact\n",
      "Loading /tmp/tmplaubk4ks/tmp.tree_36.compact\n",
      "Loading /tmp/tmplaubk4ks/tmp.tree_37.compact\n",
      "Loading /tmp/tmplaubk4ks/tmp.tree_38.compact\n",
      "Loading /tmp/tmplaubk4ks/tmp.tree_39.compact\n",
      "Loading /tmp/tmplaubk4ks/tmp.tree_4.compact\n",
      "Loading /tmp/tmplaubk4ks/tmp.tree_40.compact\n",
      "Loading /tmp/tmplaubk4ks/tmp.tree_41.compact\n",
      "Loading /tmp/tmplaubk4ks/tmp.tree_42.compact\n",
      "Loading /tmp/tmplaubk4ks/tmp.tree_43.compact\n",
      "Loading /tmp/tmplaubk4ks/tmp.tree_44.compact\n",
      "Loading /tmp/tmplaubk4ks/tmp.tree_45.compact\n",
      "Loading /tmp/tmplaubk4ks/tmp.tree_46.compact\n",
      "Loading /tmp/tmplaubk4ks/tmp.tree_47.compact\n",
      "Loading /tmp/tmplaubk4ks/tmp.tree_48.compact\n",
      "Loading /tmp/tmplaubk4ks/tmp.tree_49.compact\n",
      "Loading /tmp/tmplaubk4ks/tmp.tree_5.compact\n",
      "Loading /tmp/tmplaubk4ks/tmp.tree_6.compact\n",
      "Loading /tmp/tmplaubk4ks/tmp.tree_7.compact\n",
      "Loading /tmp/tmplaubk4ks/tmp.tree_8.compact\n",
      "Loading /tmp/tmplaubk4ks/tmp.tree_9.compact\n"
     ]
    }
   ],
   "source": [
    "import sys\n",
    "sys.path.append('../../')\n",
    "import rusty_axe.lumberjack as lumberjack\n",
    "\n",
    "\n",
    "\n",
    "selection_forest = lumberjack.fit(\n",
    "    young.X,\n",
    "    header=young.var_names,\n",
    "    p=6,\n",
    "    trees=50,\n",
    "    ifs=500,\n",
    "    ofs=500,\n",
    "    ss=300,\n",
    "    depth=9,\n",
    "    leaves=50,\n",
    "    sfr=0,\n",
    "    norm='l2',\n",
    "    dispersion_mode = \"ssme\",\n",
    "    standardize=\"true\",\n",
    "    reduction=3,\n",
    "    reduce_input='true',\n",
    "    reduce_output='false'\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saving forest\n",
      "../../data/aging_brain/selection_forest_4\n"
     ]
    }
   ],
   "source": [
    "# selection_forest.set_cache(True)\n",
    "selection_forest.backup(data_location + \"selection_forest_4\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predicting tree:49\n",
      "Fraction Unexplained:0.6650950851758846\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 56052 characters]\n",
      "text/plain": [
       "<Figure size 1800x1200 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(array(['Cldn11', 'Plp1', 'Mog', 'Mag', 'Mal', 'Ermn', 'Opalin', 'Trf',\n",
      "       'Tubb4a', 'Ppp1r14a'], dtype='<U13'), array([0.14473038, 0.16223783, 0.18386063, 0.18790939, 0.19358915,\n",
      "       0.1940862 , 0.20723898, 0.21055632, 0.21254294, 0.21817824]))\n",
      "(array(['Ccl5', 'Calca', 'Rhox5', 'Krt73', 'Cga', 'Gdf3', 'Krtdap', 'Ly6d',\n",
      "       'Fetub', 'Lhfpl1'], dtype='<U13'), array([0.97122817, 0.97142726, 0.97270366, 0.97289924, 0.97431727,\n",
      "       0.97462247, 0.97519512, 0.97554625, 0.97742106, 0.98930165]))\n",
      "[  641   553  1986  2073 13929  9579  9385  5581  5748  1659] [0.24354786 0.25451219 0.2578267  0.25990788 0.26008906 0.2619081\n",
      " 0.26712541 0.26726525 0.26827525 0.26994048]\n",
      "[ 6022  7160  1112  2410 14959 14967 14450  1655  1720 13836] [0.9120069  0.91486452 0.925852   0.93379508 0.93433898 0.94349716\n",
      " 0.94448001 0.94668649 0.951491   0.9516324 ]\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 59260 characters]\n",
      "text/plain": [
       "<Figure size 1800x1200 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "(array([0.65369459, 0.7648686 , 0.86920113, ..., 0.62990755, 0.65361374,\n",
       "        0.64946373]),\n",
       " array([0.55132951, 0.61871396, 0.66413447, ..., 0.73481175, 0.69282202,\n",
       "        0.73642703]))"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "selection_forest.self_prediction = selection_forest.predict(young.X)\n",
    "selection_forest.self_prediction.prediction_report()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "feature_residuals = selection_forest.self_prediction.residuals()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "centered = young.X - np.mean(young.X,axis=0)\n",
    "null_squared_residual = np.power(centered,2)\n",
    "feature_null = np.sum(null_squared_residual,axis=0) + 1\n",
    "\n",
    "forest_squared_residuals = np.power(feature_residuals,2)\n",
    "\n",
    "forest_feature_error = np.sum(forest_squared_residuals,axis=0) + 1\n",
    "forest_feature_remaining = forest_feature_error/feature_null\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure()\n",
    "plt.hist(forest_feature_remaining.copy(),bins=50)\n",
    "plt.show()\n",
    "\n",
    "filtered_feature_mask = forest_feature_remaining < .6\n",
    "\n",
    "print(np.sum(filtered_feature_mask))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "filtered_feature_mask = np.zeros(2000,dtype=bool)\n",
    "\n",
    "for feature in cv_forest.output_features:\n",
    "    f_i = list(young.var_names).index(feature)\n",
    "    filtered_feature_mask[f_i] = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Now we must take the predictable features and train cross-validated forests on them"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "young_filtered = young[:,filtered_feature_mask]\n",
    "young_filtered.shape\n",
    "old_filtered = old[:,filtered_feature_mask]\n",
    "old_filtered.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data_location = \"/Users/bbrener1/battle/rusty_forest_4/data/aging_brain/\"\n",
    "filtered_feature_mask = np.loadtxt(data_location + \"filtered_feature_mask.txt\")\n",
    "filtered_feature_mask = filtered_feature_mask.astype(dtype=bool)\n",
    "\n",
    "young_filtered = young[:,filtered_feature_mask]\n",
    "young_filtered.shape\n",
    "old_filtered = old[:,filtered_feature_mask]\n",
    "old_filtered.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.savetxt(data_location + \"young_filtered.tsv\",young_filtered.X)\n",
    "np.savetxt(data_location + \"old_filtered.tsv\",old_filtered.X)\n",
    "np.savetxt(data_location + \"filtered_gene_header.txt\",young_filtered.var_names,fmt='%s')\n",
    "np.savetxt(data_location + \"filtered_feature_mask.txt\", filtered_feature_mask)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Training The CV Forest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "\n",
    "import sys\n",
    "# sys.path.append('/localscratch/bbrener1/rusty_forest_v3/src')\n",
    "sys.path.append('../')\n",
    "from rusty_axe import tree_reader as tr \n",
    "from rusty_axe import lumberjack\n",
    "\n",
    "cv_forest = lumberjack.fit(\n",
    "#     young_filtered.X,\n",
    "#     header=young_filtered.var_names,\n",
    "    young.X,\n",
    "    header=young.var_names,\n",
    "    trees=100,\n",
    "    ifs=700,\n",
    "    ofs=700,\n",
    "    ss=300,\n",
    "    depth=7,\n",
    "    leaves=100,\n",
    "    dispersion_mode='ssme',\n",
    "    sfr=0,\n",
    "    standardize = 'true',\n",
    "    norm='l1',\n",
    "    reduction = 10,\n",
    "    p=8,\n",
    "    reduce_input='true',\n",
    "    reduce_output='true'\n",
    ")\n",
    "cv_forest.set_cache(True)\n",
    "cv_forest.backup(data_location + \"cv_forest_14\")\n",
    "\n",
    "\n",
    "# Should be done before 25 (15 min)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# for f in cv_forest.output_features:\n",
    "#     if \"cdk\" in f.lower():\n",
    "#         print(f)\n",
    "\n",
    "# len(cv_forest.nodes())\n",
    "# cv_forest.trim(.005)\n",
    "\n",
    "# cv_forest.trees[0].plot()\n",
    "\n",
    "# max([n.level for n in cv_forest.nodes()])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import scanpy as sc\n",
    "\n",
    "import sys\n",
    "# sys.path.append('/localscratch/bbrener1/rusty_forest_v3/src')\n",
    "sys.path.append('../')\n",
    "sys.path.append('../rusty_axe')\n",
    "from rusty_axe import tree_reader as tr \n",
    "from rusty_axe import lumberjack\n",
    "\n",
    "data_location = \"/Users/bbrener1/battle/rf_5/data/aging_brain/\"\n",
    "# data_location = \"../data/aging_brain/\"\n",
    "\n",
    "cv_forest = tr.Forest.load(data_location + 'cv_forest_trimmed_extra')\n",
    "\n",
    "# cv_forest.arguments"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# cv_forest.trim(.01)\n",
    "# len(cv_forest.nodes())\n",
    "cv_forest.output_features.shape\n",
    "# cv_forest.compute_cache()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cv_forest.reset_split_clusters()\n",
    "cv_forest.interpret_splits(\n",
    "    k=50,\n",
    "    pca=100,\n",
    "    depth=10,\n",
    "    metric=\"cosine\",\n",
    "    mode='partial',\n",
    "    relatives=True\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cv_forest.maximum_spanning_tree(mode='samples')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "cv_forest.tsne_coordinates = filtered.obsm['X_umap'][young_mask]\n",
    "cv_forest.html_tree_summary(n=5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cv_forest.self_prediction = cv_forest.predict(cv_forest.output)\n",
    "cv_forest.self_prediction.node_sample_encoding()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cv_forest.self_prediction.prediction_report()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# cv_forest.old_prediction = cv_forest.predict(old.X.T[filtered_feature_mask].T)\n",
    "# cv_forest.old_prediction.prediction_report()\n",
    "data_location"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Comparing Different Mouse Partitions"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Comparing Features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# for i,partition in enumerate(partitions):\n",
    "#     partition = np.array(partition).astype(dtype=bool)\n",
    "#     partition_mask = np.any(batch_encoding[young_mask,:8][:,partition],axis=1)\n",
    "#     sample_indices = np.arange(young_filtered.X.shape[0])[partition_mask]\n",
    "#     sub_forest = cv_forest.derive_samples(sample_indices)\n",
    "#     sub_forest.backup(data_location + f\"sub_forest_{i}\")\n",
    "#     del(sub_forest)\n",
    "\n",
    "for i,partition in list(enumerate(partitions)):\n",
    "    partition = np.array(partition).astype(dtype=bool)\n",
    "    partition_mask = np.any(batch_encoding[young_mask,:8][:,partition],axis=1)\n",
    "    sub_forest = tr.Forest.load(data_location + f\"sub_forest_{i}\")\n",
    "    sub_forest.self_prediction = sub_forest.predict(young_filtered.X[partition_mask])\n",
    "    sub_forest.self_prediction.node_sample_encoding()\n",
    "    sub_forest.self_prediction.node_sample_r2()\n",
    "    sub_forest.test_prediction = sub_forest.predict(young_filtered.X[~partition_mask])\n",
    "    sub_forest.test_prediction.node_sample_encoding()\n",
    "    sub_forest.test_prediction.node_sample_r2()\n",
    "    sub_forest.backup(data_location + f\"sub_forest_{i}_cached\")\n",
    "    del(sub_forest)\n",
    "\n",
    "# determination_spread = np.zeros((39,20))\n",
    "# self_determination_spread = np.zeros((39,20))\n",
    "# other_determination_spread = np.zeros((39,20))\n",
    "\n",
    "# for i,partition in enumerate(partitions[:21]):\n",
    "#     print(\"+++++++++++++++++++\")\n",
    "#     print(i)\n",
    "#     print(\"+++++++++++++++++++\")\n",
    "#     partition = np.array(partition).astype(dtype=bool)\n",
    "#     partition_mask = np.any(batch_encoding[young_mask,:8][:,partition],axis=1)\n",
    "#     sub_forest = tr.Forest.load(data_location + \"restricted_sub_forest/\" + f\"sub_forest_{i}_cached\")\n",
    "#     for factor_index in range(1,39):\n",
    "#         factor = sub_forest.split_clusters[factor_index]\n",
    "#         self_fvu,other_fvu,_ = sub_forest.self_prediction.compare_factor_fvu(sub_forest.test_prediction,factor)\n",
    "#         other_determination_spread[factor_index,i] = 1-other_fvu\n",
    "#         delta = other_fvu - self_fvu\n",
    "#         determination_spread[factor_index,i] = delta\n",
    "#     del(sub_forest)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mean_spread = np.mean(determination_spread,axis=1)\n",
    "spread_variance = np.var(determination_spread,axis=1)\n",
    "mean_sort = np.argsort(mean_spread)\n",
    "var_sort = np.argsort(spread_variance)\n",
    "\n",
    "plt.figure()\n",
    "for i in range(39):\n",
    "    plt.scatter(np.array([i,]*20),determination_spread[var_sort[i]],s=2,c='blue')\n",
    "plt.scatter(np.arange(39),mean_spread[var_sort],c='red')\n",
    "plt.xticks(np.arange(39),var_sort,rotation=90)\n",
    "plt.show()    \n",
    "\n",
    "\n",
    "plt.figure()\n",
    "plt.title(\"Mean Prediction Error Young vs Young and Young vs Old\")\n",
    "for i in range(39):\n",
    "    plt.scatter(np.array([i,]*20),determination_spread[mean_sort[i]],s=2,c='blue')\n",
    "plt.scatter(np.arange(39),mean_spread[mean_sort],c='red')\n",
    "plt.xticks(np.arange(39),mean_sort,rotation=90)\n",
    "plt.show()    \n",
    "\n",
    "print(list(enumerate(np.sqrt(spread_variance))))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "for factor in cv_forest.split_clusters[1:]:\n",
    "    print(\"+++++++++++++++++++++\")\n",
    "    print(f\"Factor {factor.name()}\")\n",
    "    print(\"+++++++++++++++++++++\")\n",
    "    cv_forest.self_prediction.compare_factor_fvu(cv_forest.old_prediction, factor)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "len(cv_forest.split_clusters)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "self_fvu = [1.,]\n",
    "old_fvu = [1.,]\n",
    "\n",
    "for factor in cv_forest.split_clusters[1:]:\n",
    "    print(\"++++++++++++++++++++\")\n",
    "    print(factor.name())\n",
    "    print(\"++++++++++++++++++++\")\n",
    "    (sfvu,olfvu,olmwu) = cv_forest.self_prediction.compare_factor_fvu(cv_forest.old_prediction,factor,plot=True)\n",
    "    self_fvu.append(sfvu)\n",
    "    old_fvu.append(olfvu)\n",
    "\n",
    "self_cod = 1 - np.array(self_fvu) \n",
    "old_cod = 1 - np.array(old_fvu)\n",
    "# other_cod = 1 - np.array(other_fvu)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "old_delta = self_cod - old_cod\n",
    "delta_sort = np.argsort(old_delta)\n",
    "\n",
    "print(list(zip(delta_sort,old_delta[delta_sort])))\n",
    "\n",
    "# selected_labels = set([7,27,16,35])\n",
    "\n",
    "plt.figure()\n",
    "plt.title(\"Coefficient of Determination Ratio\")\n",
    "plt.plot([0,.5],[0,.5],label=\"Ideal Fit\", color='red')\n",
    "for i,(p1,p2) in enumerate(zip(self_cod,old_cod)):\n",
    "        plt.text(p1+.005,p2-.01,str(i),fontsize=5)\n",
    "\n",
    "#     if i in selected_labels:\n",
    "#         plt.text(p1+.005,p2-.01,str(i),fontsize=10)\n",
    "#         plt.scatter([p1,],[p2,],s=10,color='blue')\n",
    "plt.scatter(self_cod,old_cod,label=\"Old Mice\",s=2,color='blue')\n",
    "plt.legend()\n",
    "plt.xlabel(\"Trained COD\")\n",
    "plt.ylabel(\"Observed COD\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "old_delta = self_cod - old_cod\n",
    "delta_sort = np.argsort(old_delta)\n",
    "\n",
    "cod_sort = np.argsort(self_cod)\n",
    "\n",
    "# self_min = self_cod - (1.5*mean_spread)\n",
    "# self_max = self_cod + (1.5*mean_spread)\n",
    "\n",
    "selected_labels = set([7,27,14,1,22])\n",
    "\n",
    "\n",
    "print(list(zip(delta_sort,old_delta[delta_sort])))\n",
    "\n",
    "plt.figure(figsize=(4,3))\n",
    "plt.title(\"Coefficient of Determination Ratio, Young vs Old\")\n",
    "plt.plot([0,.5],[0,.5],label=\"Ideal Fit\", color='red')\n",
    "for i,(p1,p2) in enumerate(zip(self_cod,old_cod)):\n",
    "    if i in selected_labels:\n",
    "        plt.text(p1+.003,p2-.005,str(i),fontsize=10)\n",
    "        plt.scatter([p1,],[p2,],color='green',s=10)\n",
    "plt.scatter([],[],color='green',label='Selected Factors')\n",
    "# plt.fill_between(self_cod[cod_sort],self_min[cod_sort],self_max[cod_sort])\n",
    "plt.scatter(self_cod,old_cod,s=2)\n",
    "plt.legend()\n",
    "plt.xlabel(\"Trained COD\")\n",
    "plt.ylabel(\"Observed COD\")\n",
    "plt.xlim(0,.4)\n",
    "plt.ylim(0,.4)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "selected_labels = set([7,27,14,1,22])\n",
    "\n",
    "plt.figure(figsize=(5,4))\n",
    "plt.title(\"Mean Prediction Discrepancy,\\n Young vs Young and Young vs Old\")\n",
    "for i in range(39):\n",
    "    plt.scatter(np.array([i,]*20),determination_spread[mean_sort[i]],s=2,c='blue')\n",
    "\n",
    "spread_min = np.min(determination_spread.flatten())    \n",
    "for i in range(39):\n",
    "    if mean_sort[i] in selected_labels:\n",
    "        plt.scatter([i,],[spread_min,],marker=\"*\",color='black')\n",
    "plt.scatter([],[],color='black',label='Significant')\n",
    "plt.scatter([],[],color='blue',label=\"Bootstrapped Young vs Young Δ COD\") #phantom scatter to apply label of blue dots\n",
    "plt.scatter(np.arange(39),mean_spread[mean_sort],c='red',label=\"Mean Δ COD Young vs Young\")\n",
    "plt.scatter(np.arange(39),old_delta[mean_sort],c='green',label=\"Mean Δ COD Young vs Old\")\n",
    "plt.xticks(np.arange(39),mean_sort,rotation=90,fontsize=8)\n",
    "plt.xlabel(\"Factor\")\n",
    "plt.ylabel(\"Difference in Coefficient of Determination\")\n",
    "plt.legend()\n",
    "plt.show()    \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mean_determination = np.mean(other_determination_spread,axis=1)\n",
    "mean_sort = np.argsort(mean_determination)\n",
    "\n",
    "selected_labels = set([7,27,14,1,22])\n",
    "\n",
    "plt.figure(figsize=(4,3))\n",
    "plt.title(\"Factor COD\",fontsize=16)\n",
    "for i in range(39):\n",
    "    plt.scatter(np.array([i,]*20),other_determination_spread[mean_sort[i]],s=1,c='blue')\n",
    "    \n",
    "for i in range(39):\n",
    "    if mean_sort[i] in selected_labels:\n",
    "        plt.scatter([i,],[spread_min,],marker=\"*\",color='black')\n",
    "plt.scatter([],[],color='black',label='Significant')\n",
    "\n",
    "plt.scatter(np.arange(39),mean_determination[mean_sort],c='red',label=\"Young vs Young (Mean)\",alpha=.5)\n",
    "plt.scatter(np.arange(39),old_cod[mean_sort],label=\"Young Vs Old\",c='green',alpha=.5)\n",
    "plt.scatter([],[],color='blue',label=\"Bootstraps\")\n",
    "plt.xticks(np.arange(39),mean_sort,rotation=90,fontsize=8)\n",
    "plt.legend()\n",
    "plt.ylabel(\"COD\")\n",
    "plt.xlabel(\"Factor\")\n",
    "plt.show()    \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(mean_determination[13])\n",
    "print(old_cod[13])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "f_i = 22\n",
    "print(mean_determination[f_i])\n",
    "print(np.min(other_determination_spread[f_i]))\n",
    "print(np.max(other_determination_spread[f_i]))\n",
    "print(old_cod[f_i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "delta_z = np.abs(old_delta/np.sqrt(spread_variance))\n",
    "z_sort = np.argsort(delta_z[1:])\n",
    "z_sort + 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(list(zip(z_sort+1,delta_z[1:][z_sort])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Volcano plot plotting differences in COD by significance and magnitude. \n",
    "\n",
    "selected_labels = set([7,27,16,1,30,14,22,29])\n",
    "\n",
    "plt.figure(figsize=(3.5,3))\n",
    "plt.title(\"Δ COD vs Z Score\")\n",
    "plt.scatter(old_delta[1:],delta_z[1:],s=10)\n",
    "for i,(p1,p2) in enumerate(zip(old_delta[1:],delta_z[1:])):\n",
    "    if i+1 in selected_labels:\n",
    "        plt.text(p1+.003,p2-.005,str(i+1),fontsize=8)\n",
    "plt.ylabel(\"Z Score\")\n",
    "plt.xlabel(\"Δ COD\")\n",
    "plt.plot([-.14,.14,],[3.3,3.3],'--',label='Significance: \\n P > .001',color='red')\n",
    "plt.plot([.02,.02,],[0,16],'--',color='green',label='ΔCOD > 2%')\n",
    "plt.plot([-.02,-.02,],[0,16],'--',color='lightgray',label='ΔCOD < 2%')\n",
    "plt.xlim(-.14,.14)\n",
    "plt.legend(fontsize=8)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from matplotlib.colors import DivergingNorm\n",
    "\n",
    "factor = cv_forest.split_clusters[7]\n",
    "sister_scores = factor.sister_scores()\n",
    "\n",
    "fig = plt.figure(figsize=(3,2.5))\n",
    "plt.title(\n",
    "    f\"Distribution of Samples \\nIn {factor.name()} (Red) vs Its Sisters (Blue)\")\n",
    "plt.scatter(*cv_forest.tsne_coordinates.T, s=1,\n",
    "            alpha=.6, c=sister_scores, norm=DivergingNorm(0), cmap='bwr')\n",
    "plt.colorbar(label=\"Sister Score\")\n",
    "plt.ylabel(\"tSNE Coordinates (AU)\")\n",
    "plt.xlabel(\"tSNE Coordinates (AU)\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Comparing Cell Type Populations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "self_fraction_spread = np.zeros((39,20))\n",
    "other_fraction_spread = np.zeros((39,20))\n",
    "\n",
    "for i,partition in enumerate(partitions[:20]):\n",
    "    print(\"+++++++++++++++++++\")\n",
    "    print(i)\n",
    "    print(\"+++++++++++++++++++\")\n",
    "    partition = np.array(partition).astype(dtype=bool)\n",
    "    partition_mask = np.any(batch_encoding[young_mask,:8][:,partition],axis=1)\n",
    "    sub_forest = tr.Forest.load(data_location + \"restricted_sub_forest/\" + f\"sub_forest_{i}_cached\")\n",
    "    for factor_index in range(1,39):\n",
    "        factor = sub_forest.split_clusters[factor_index]\n",
    "        self_fraction,other_fraction,_ = sub_forest.self_prediction.compare_factor_fractions(sub_forest.test_prediction,factor)\n",
    "        self_fraction_spread[factor_index,i] = self_fraction\n",
    "        other_fraction_spread[factor_index,i] = other_fraction\n",
    "    del(sub_forest)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mean_fraction = np.mean(other_fraction_spread,axis=1)\n",
    "fraction_sort = np.argsort(mean_fraction)\n",
    "\n",
    "plt.figure(figsize=(5,4))\n",
    "plt.title(\"Factor Fraction\")\n",
    "for i in range(1,39):\n",
    "    plt.scatter(np.array([i,]*20),other_fraction_spread[fraction_sort[i]],s=1,c='blue')\n",
    "    \n",
    "for i in range(1,39):\n",
    "    if fraction_sort[i] in selected_labels:\n",
    "        plt.scatter([i,],[.3,],marker=\"*\",color='black')\n",
    "\n",
    "plt.scatter([],[],color='black',label='Significant')\n",
    "\n",
    "plt.scatter(np.arange(1,39),mean_fraction[fraction_sort[1:]],c='red',label=\"Young vs Young (Mean)\",alpha=.5)\n",
    "plt.scatter([],[],color='blue',label=\"Bootstraps\")\n",
    "plt.scatter(np.arange(1,39),np.array(old_fractions)[fraction_sort[1:]],label=\"Old Fraction\")\n",
    "plt.xticks(np.arange(1,39),fraction_sort[1:],rotation=90,fontsize=8)\n",
    "plt.legend()\n",
    "plt.ylabel(\"Fraction\")\n",
    "plt.xlabel(\"Factor\")\n",
    "plt.show()    \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "old_fractions = [0,]\n",
    "for factor in cv_forest.split_clusters[1:]:\n",
    "    self_fraction,old_fraction,_ = cv_forest.self_prediction.compare_factor_fractions(cv_forest.old_prediction,factor)\n",
    "    old_fractions.append(old_fraction)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mouse_ratios = np.zeros((39,8))\n",
    "\n",
    "\n",
    "for i,mouse in enumerate(batch_encoding[young_mask].T[:8]):\n",
    "    for j,factor in enumerate(cv_forest.split_clusters[1:]):\n",
    "        node_encoding = cv_forest.node_sample_encoding(factor.nodes)[mouse]\n",
    "        parent_encoding = cv_forest.node_sample_encoding(factor.parents())[mouse]\n",
    "#         node_pop = np.sum(node_encoding,axis=0)\n",
    "#         parent_pop = np.sum(parent_encoding,axis=0)\n",
    "#         ratio = (node_pop+1)/(parent_pop+1)\n",
    "#         mean_ratio = np.mean(ratio)\n",
    "        node_pop = np.sum(node_encoding)\n",
    "        parent_pop = np.sum(parent_encoding)\n",
    "        mean_ratio = (node_pop+1)/(parent_pop+1)\n",
    "        mouse_ratios[j+1,i] = mean_ratio\n",
    "\n",
    "old_mouse_ratios = np.zeros((39,8))\n",
    "\n",
    "\n",
    "for i,mouse in enumerate(batch_encoding[old_mask].T[8:]):\n",
    "    for j,factor in enumerate(cv_forest.split_clusters[1:]):\n",
    "        nodes = [n for n in factor.nodes if n.parent is not None]\n",
    "        parents = [n.parent for n in nodes]\n",
    "        node_indices = [n.index for n in nodes]\n",
    "        parent_indices = [p.index for p in parents]\n",
    "        node_encoding = cv_forest.old_prediction.node_sample_encoding()[node_indices].T[mouse]\n",
    "        parent_encoding = cv_forest.old_prediction.node_sample_encoding()[parent_indices].T[mouse]\n",
    "#         node_pop = np.sum(node_encoding,axis=0)\n",
    "#         parent_pop = np.sum(parent_encoding,axis=0)\n",
    "#         ratio = (node_pop+1)/(parent_pop+1)\n",
    "#         mean_ratio = np.mean(ratio)\n",
    "        node_pop = np.sum(node_encoding)\n",
    "        parent_pop = np.sum(parent_encoding)\n",
    "        mean_ratio = (node_pop+1)/(parent_pop+1)\n",
    "        old_mouse_ratios[j+1,i] = mean_ratio\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "selected_labels = set([27,34,13,21,6,7])\n",
    "\n",
    "plt.figure(figsize=(14,2.5))\n",
    "plt.title(\"Factor fraction, young vs old\")\n",
    "for mouse in mouse_ratios.T:\n",
    "    plt.scatter(np.arange(38)-.1,mouse[fraction_sort][1:],s=2,c='blue')\n",
    "for mouse in old_mouse_ratios.T:\n",
    "    plt.scatter(np.arange(38)+.1,mouse[fraction_sort][1:],s=2,c='red')\n",
    "for i,fs in list(enumerate(fraction_sort))[1:]:\n",
    "    if fs in selected_labels:\n",
    "        plt.scatter([i-1,],.1,marker=\"*\",c='black')\n",
    "plt.scatter([],[],c='blue',label=\"Young\")\n",
    "plt.scatter([],[],c='red',label='Old')\n",
    "plt.scatter([],[],c='black',label='Significant')\n",
    "for i in range(38):\n",
    "    plt.plot([i+.5,i+.5],[.1,.9],\"--\",linewidth=.5,color='lightgray')\n",
    "\n",
    "plt.xticks(np.arange(38),labels=fraction_sort[1:],rotation=90,fontsize=10)\n",
    "plt.legend()\n",
    "plt.xlabel(\"Factors\")\n",
    "plt.ylabel(\"Fraction of Parent\")\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from scipy.stats import mannwhitneyu\n",
    "\n",
    "ratio_mwus = []\n",
    "\n",
    "for i in range(1,39):\n",
    "    mwu = mannwhitneyu(mouse_ratios[i],old_mouse_ratios[i])\n",
    "    print(\"++++++++++++++++++\")\n",
    "    print(i)\n",
    "    print(mwu)\n",
    "    print(\"++++++++++++++++++\")\n",
    "    ratio_mwus.append(mwu)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mwu_sort = np.argsort([rm[1] for rm in ratio_mwus]) + 1\n",
    "\n",
    "for i in range(1,38):\n",
    "    print(mwu_sort[i])\n",
    "    print(ratio_mwus[mwu_sort[i]-1])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Comparing Prediction Within Individual Clusters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "young_self_error,young_parent_error = cv_forest.self_prediction.factor_feature_error(cv_forest.split_clusters[7])\n",
    "old_self_error,old_parent_error = cv_forest.old_prediction.factor_feature_error(cv_forest.split_clusters[7])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "young_feature_cod = 1-(young_self_error/young_parent_error)\n",
    "old_feature_cod = 1-(old_self_error/old_parent_error)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "old_cod_sort = np.argsort(old_feature_cod)\n",
    "\n",
    "plt.figure(figsize=(3.5,3.5))\n",
    "plt.title(\"COD In Young vs Old \\n Factor 7\")\n",
    "plt.scatter(young_feature_cod[old_cod_sort[1:]],old_feature_cod[old_cod_sort[1:]],s=2)\n",
    "plt.xlabel(\"COD Young\")\n",
    "plt.ylabel(\"COD Old\")\n",
    "plt.plot([0,.3],[0,.3],color='red',label=\"Ideal Fit\")\n",
    "plt.plot([0,.5],[0,0],color='gray')\n",
    "plt.plot([.15,.15],[.3,-.3],\"--\",color='red',label=\"Good Predictive Power\")\n",
    "plt.legend(loc='lower left',fontsize=7)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "delta_cod = old_feature_cod - young_feature_cod\n",
    "delta_cod_sort = np.argsort(delta_cod)\n",
    "\n",
    "important_features = [x for x in zip(zip(cv_forest.output_features[delta_cod_sort],delta_cod[delta_cod_sort]),young_feature_cod[delta_cod_sort]) if x[1]>.15]\n",
    "print(important_features)\n",
    "\n",
    "for ((feature,delta),original) in important_features:\n",
    "    print(f\"{feature}:\\t\\t{original}\\t -> \\t{original+delta}\")\n",
    "    \n",
    "    \n",
    "# Important results: Mobp, Plp1 stay accurate,   Pdlim2, Mog collapse"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len([x[0][0] for x in important_features])\n",
    "\n",
    "important_fi = [list(cv_forest.output_features).index(f[0][0]) for f in important_features]\n",
    "\n",
    "local_important = cv_forest.split_clusters[7].local_correlations(indices=important_fi)\n",
    "global_important = cv_forest.global_correlations(indices=important_fi)\n",
    "\n",
    "from scipy.cluster.hierarchy import linkage,dendrogram\n",
    "\n",
    "local_agg = dendrogram(linkage(local_important, metric='cosine', method='average'), no_plot=True)['leaves']\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "plt.figure()\n",
    "plt.imshow(local_important,cmap='bwr',vmin=-1,vmax=1)\n",
    "plt.show()\n",
    "\n",
    "plt.figure()\n",
    "plt.imshow(local_important[local_agg].T[local_agg],cmap='bwr',vmin=-1,vmax=1)\n",
    "plt.show()\n",
    "\n",
    "# plt.figure()\n",
    "# plt.imshow(global_important[].T[local_agg],cmap='bwr',vmin=-1,vmax=1)\n",
    "# plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# from scipy.cluster.hierarchy import linkage,dendrogram\n",
    "\n",
    "# feature_agg = dendrogram(linkage(young_filtered.X.T, metric='cosine', method='average'), no_plot=True)['leaves']\n",
    "# young_agg = dendrogram(linkage(young_filtered.X, metric='cosine', method='average'), no_plot=True)['leaves']\n",
    "# old_agg = dendrogram(linkage(old_filtered.X, metric='cosine', method='average'), no_plot=True)['leaves']\n",
    "\n",
    "young_factors = cv_forest.self_prediction.factor_matrix()\n",
    "old_factors = cv_forest.old_prediction.factor_matrix()\n",
    "\n",
    "# factor_agg = dendrogram(linkage(young_factors.T[1:], metric='cosine', method='average'), no_plot=True)['leaves']\n",
    "\n",
    "plt.figure()\n",
    "plt.imshow(young_filtered.X[young_agg].T[feature_agg].T,aspect='auto',interpolation='none')\n",
    "plt.xticks(np.arange(0,351,10),rotation=(60))\n",
    "plt.show()\n",
    "\n",
    "plt.figure()\n",
    "plt.imshow(young_factors[young_agg].T[1:][factor_agg].T,aspect='auto',interpolation='none',cmap='bwr',vmin=-1,vmax=1)\n",
    "plt.xticks(np.arange(38),labels=np.array(factor_agg)+1,rotation=60)\n",
    "plt.show()\n",
    "\n",
    "\n",
    "plt.figure()\n",
    "plt.imshow(old_filtered.X[old_agg].T[feature_agg].T,aspect='auto',interpolation='none')\n",
    "plt.show()\n",
    "\n",
    "plt.figure()\n",
    "plt.imshow(old_factors[old_agg].T[1:][factor_agg].T,aspect='auto',interpolation='none',cmap='bwr',vmin=-1,vmax=1)\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print([young_filtered.var_names[fi] for fi in feature_agg[185:205]])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## TESTING"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ci = 22\n",
    "cluster = cv_forest.split_clusters[ci]\n",
    "partials = np.mean(cv_forest.partial_matrix(cluster.sisters()),axis=0)\n",
    "\n",
    "partial_sort = np.argsort(np.abs(partials))\n",
    "\n",
    "print(cv_forest.output_features[partial_sort[-10:]])\n",
    "\n",
    "partials[partial_sort[-10:]]\n",
    "\n",
    "feature = 'mt-Nd1'\n",
    "feature_index = list(cv_forest.output_features).index(feature)\n",
    "\n",
    "print(partials[feature_index])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cv_forest.split_clusters[5].plot_sister_scores()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "[s.level for s in cv_forest.split_clusters[23].sisters()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.mean(cv_forest.node_representation(cv_forest.split_clusters[22].nodes,mode='additive_mean'),axis=0)[feature_index]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "plt.figure()\n",
    "plt.scatter(*filtered[young_mask].obsm['X_umap'].T,s=1)\n",
    "plt.show()\n",
    "\n",
    "encoding = cv_forest.node_sample_encoding(cv_forest.split_clusters[3].nodes)\n",
    "scores = np.sum(encoding.T,axis=0)\n",
    "scores.shape\n",
    "\n",
    "plt.figure()\n",
    "plt.scatter(*filtered[young_mask].obsm['X_umap'].T,s=1,c=scores)\n",
    "plt.show()\n",
    "\n",
    "sister_encoding = cv_forest.node_sample_encoding(cv_forest.split_clusters[3].sisters())\n",
    "sister_scores = np.sum(sister_encoding.T,axis=0)\n",
    "\n",
    "plt.figure()\n",
    "plt.scatter(*filtered[young_mask].obsm['X_umap'].T,s=1,c=sister_scores)\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "young_batch = batch_encoding[young_mask][:,:8]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure()\n",
    "plt.imshow(young_batch,aspect='auto')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.sum(young_batch,axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "partition = partitions[5]\n",
    "\n",
    "train = np.zeros(16028,dtype=bool)\n",
    "test = np.zeros(16028,dtype=bool)\n",
    "\n",
    "for i,m in enumerate(partition):\n",
    "    if m:\n",
    "        train[young_batch[:,i]] = True\n",
    "    else:\n",
    "        test[young_batch[:,i]] = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure()\n",
    "plt.imshow(train.reshape([-1,1]),aspect='auto')\n",
    "plt.show()\n",
    "\n",
    "plt.figure()\n",
    "plt.imshow(test.reshape([-1,1]),aspect='auto')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "sys.path.append('../../')\n",
    "import rusty_axe.lumberjack as lumberjack\n",
    "\n",
    "\n",
    "\n",
    "validation_forest = lumberjack.fit(\n",
    "    young.X[train],\n",
    "    header=young.var_names,\n",
    "    trees=100,\n",
    "    ifs=700,\n",
    "    ofs=700,\n",
    "    ss=300,\n",
    "    depth=8,\n",
    "    leaves=10,\n",
    "    sfr=0,\n",
    "    norm='l1',\n",
    "    dispersion_mode = \"ssme\",\n",
    "    reduce_input='true',\n",
    "    reduce_output='false'\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "validation_forest.set_cache(True)\n",
    "validation_forest.backup(data_location+\"validation_forest\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "validation_forest.self_prediction = validation_forest.predict(young.X[train])\n",
    "validation_forest.self_prediction.prediction_report()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "validation_forest.other_prediction = validation_forest.predict(young.X[test])\n",
    "validation_forest.other_prediction.prediction_report()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
