{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "# Accuracy of BMLiNGAM\n", "This notebook shows accuracy of BMLiNGAM for causal inference." ] }, { "cell_type": "code", "execution_count": 1, "metadata": { "collapsed": false }, "outputs": [ { "data": { "application/javascript": [ "IPython.notebook.set_autosave_interval(0)" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "Autosave disabled\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "/opt/conda/lib/python3.5/site-packages/matplotlib/__init__.py:1357: UserWarning: This call to matplotlib.use() has no effect\n", "because the backend has already been chosen;\n", "matplotlib.use() must be called *before* pylab, matplotlib.pyplot,\n", "or matplotlib.backends is imported for the first time.\n", "\n", " warnings.warn(_use_error_msg)\n" ] } ], "source": [ "%matplotlib inline\n", "%autosave 0\n", "import sys, os\n", "sys.path.insert(0, os.path.expanduser('~/work/git/github/taku-y/bmlingam'))\n", "sys.path.insert(0, os.path.expanduser('~/work/git/github/pymc-devs/pymc3'))\n", "\n", "import theano\n", "theano.config.floatX = 'float64'\n", "\n", "from copy import deepcopy\n", "import hashlib\n", "import matplotlib.pyplot as plt\n", "import numpy as np\n", "import pandas as pd\n", "import seaborn as sns\n", "import time\n", "\n", "from expr1 import run_trial\n", "from bmlingam import load_pklz, save_pklz\n", "# from bmlingam import do_mcmc_bmlingam, InferParams, MCMCParams, save_pklz, load_pklz, define_hparam_searchspace, find_best_model\n", "# from bmlingam.utils.gendata import GenDataParams, gen_artificial_data" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Experimental condition\n", "The experimental conditions are as follows.\n", "\n", "* Parameter of artiifcial data\n", "\n", " * The number of observed samples (`n_samples`): `[100]`\n", " * The number of confounding factors (`n_confs` or $Q$): `[1, 3, 5, 10]`\n", " * Scale of total noise: $c=0.25, 0.5, 1.0, 3.0$. \n", " * Scale of confounding factors: $c/\\sqrt{Q}$\n", " * Distribution of observation noise (`data_noise_type`): `['laplace', 'uniform']`\n", " * Scale of observation noise: 3\n", "\n", "* Hyperparameter range\n", "\n", " * Correlation coefficient of confounding factors: (`L_cov_21s`): `[[-.9, -.7, -.5, -.3, 0, .3, .5, .7, .9]]`\n", " * Distribution of observation noise (`model_noise_type`): `['gg']`" ] }, { "cell_type": "code", "execution_count": 2, "metadata": { "collapsed": true }, "outputs": [], "source": [ "conds = [\n", " {\n", " 'totalnoise': totalnoise, \n", " 'L_cov_21s': L_cov_21s, \n", " 'n_samples': n_samples, \n", " 'n_confs': n_confs, \n", " 'data_noise_type': data_noise_type, \n", " 'model_noise_type': model_noise_type\n", " }\n", " for totalnoise in [0.25, 0.5, 1.0, 3.0]\n", " for L_cov_21s in [[-.9, -.7, -.5, -.3, 0, .3, .5, .7, .9]]\n", " for n_samples in [100]\n", " for n_confs in [1, 3, 5, 10] # [1, 3, 5, 10]\n", " for data_noise_type in ['laplace', 'uniform']\n", " for model_noise_type in ['gg']\n", " ]" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Program" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Identifier of trial\n", "Identifier of a trial is determined based on:\n", "\n", "* Trial index (`ix_trial`)\n", "* The number of samples (`n_samples`)\n", "* The number of confounders (`n_confs`)\n", "* Distribution of observation noise of artificial data (`data_noise_type`)\n", "* Distribution of observation noise of BMLiNGAM model (`model_noise_type`)\n", "* Correlation coefficient of confounding effect (`L_cov_21s`)\n", "* Total noise scale (`totalnoise`)\n", "\n", "We use identifiers to store results of trials." ] }, { "cell_type": "code", "execution_count": 3, "metadata": { "collapsed": false }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "8eb07d5dcb68e8163a7a2a0582b59d1f\n" ] } ], "source": [ "def make_id(ix_trial, n_samples, n_confs, data_noise_type, model_noise_type, L_cov_21s, totalnoise):\n", " L_cov_21s_ = ' '.join([str(v) for v in L_cov_21s])\n", " \n", " return hashlib.md5(\n", " str((L_cov_21s_, ix_trial, n_samples, n_confs, data_noise_type, model_noise_type, totalnoise)).encode('utf-8')\n", " ).hexdigest()\n", "\n", "# Test\n", "print(make_id(55, 100, 12, 'all', 'gg', [1, 2, 3], 0.3))" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Append results to dataframe" ] }, { "cell_type": "code", "execution_count": 4, "metadata": { "collapsed": false }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "--- df1 ---\n", " col1 col2\n", "0 10 20\n", "--- df2 ---\n", " col1 col2\n", "0 10 20\n", "1 30 -10\n" ] } ], "source": [ "def add_result_to_df(df, result):\n", " if df is None:\n", " return pd.DataFrame({k: [v] for k, v in result.items()})\n", " else:\n", " return df.append(result, ignore_index=True)\n", "\n", "# Test\n", "result1 = {'col1': 10, 'col2': 20}\n", "result2 = {'col1': 30, 'col2': -10}\n", "df1 = add_result_to_df(None, result1)\n", "print('--- df1 ---')\n", "print(df1)\n", "df2 = add_result_to_df(df1, result2)\n", "print('--- df2 ---')\n", "print(df2)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Save and load data frame" ] }, { "cell_type": "code", "execution_count": 5, "metadata": { "collapsed": true }, "outputs": [], "source": [ "def load_df(df_file):\n", " if os.path.exists(df_file):\n", " return load_pklz(df_file)\n", " else:\n", " return None\n", "\n", "def save_df(df_file, df):\n", " save_pklz(df_file, df)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Perform experiment over conditions and trials" ] }, { "cell_type": "code", "execution_count": 6, "metadata": { "collapsed": false }, "outputs": [], "source": [ "def df_exist_result_id(df, result_id):\n", " if df is not None:\n", " return result_id in np.array(df['result_id'])\n", " else:\n", " False\n", "\n", "def run_expr(conds, n_trials_per_cond=50):\n", " \"\"\"Perform evaluation of BMLiNGAM given a set of experimental conditions.\n", " \n", " For each condition, several trials are executed.\n", " In a trial, BMLiNGAM is applied to causal inference for artificial data.\n", " The average accuracy is computed for each condition.\n", " \"\"\"\n", " # Filename of dataframe\n", " data_dir = '.'\n", " df_file = data_dir + '/20160822-eval-bml-results.pklz'\n", " \n", " # Load results computed in previous\n", " df = load_df(df_file)\n", "\n", " # Loop over experimental conditions\n", " n_skip = 0\n", " for cond in conds:\n", " print(cond)\n", " \n", " # Loop over trials\n", " for ix_trial in range(n_trials_per_cond):\n", " # Identifier of a trial for (cond, ix_trial)\n", " result_id = make_id(ix_trial, **cond)\n", " \n", " # Check if the result has been already stored in the data frame\n", " if df_exist_result_id(df, result_id):\n", " n_skip += 1\n", " else:\n", " # `result` is a dict including results of trials.\n", " # `ix_trial` is used as the random seed of the corresponding trial. \n", " result = run_trial(ix_trial, cond)\n", " result.update({'result_id': result_id})\n", " \n", " df = add_result_to_df(df, result)\n", " save_df(df_file, df)\n", " \n", " print('Number of skipped trials = {}'.format(n_skip))\n", " return df" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Run program" ] }, { "cell_type": "code", "execution_count": 7, "metadata": { "collapsed": false, "scrolled": false }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "{'totalnoise': 0.25, 'L_cov_21s': [-0.9, -0.7, -0.5, -0.3, 0, 0.3, 0.5, 0.7, 0.9], 'n_confs': 1, 'model_noise_type': 'gg', 'data_noise_type': 'laplace', 'n_samples': 100}\n", "{'totalnoise': 0.25, 'L_cov_21s': [-0.9, -0.7, -0.5, -0.3, 0, 0.3, 0.5, 0.7, 0.9], 'n_confs': 1, 'model_noise_type': 'gg', 'data_noise_type': 'uniform', 'n_samples': 100}\n", "{'totalnoise': 0.25, 'L_cov_21s': [-0.9, -0.7, -0.5, -0.3, 0, 0.3, 0.5, 0.7, 0.9], 'n_confs': 3, 'model_noise_type': 'gg', 'data_noise_type': 'laplace', 'n_samples': 100}\n", "{'totalnoise': 0.25, 'L_cov_21s': [-0.9, -0.7, -0.5, -0.3, 0, 0.3, 0.5, 0.7, 0.9], 'n_confs': 3, 'model_noise_type': 'gg', 'data_noise_type': 'uniform', 'n_samples': 100}\n", "{'totalnoise': 0.25, 'L_cov_21s': [-0.9, -0.7, -0.5, -0.3, 0, 0.3, 0.5, 0.7, 0.9], 'n_confs': 5, 'model_noise_type': 'gg', 'data_noise_type': 'laplace', 'n_samples': 100}\n", "{'totalnoise': 0.25, 'L_cov_21s': [-0.9, -0.7, -0.5, -0.3, 0, 0.3, 0.5, 0.7, 0.9], 'n_confs': 5, 'model_noise_type': 'gg', 'data_noise_type': 'uniform', 'n_samples': 100}\n", "{'totalnoise': 0.25, 'L_cov_21s': [-0.9, -0.7, -0.5, -0.3, 0, 0.3, 0.5, 0.7, 0.9], 'n_confs': 10, 'model_noise_type': 'gg', 'data_noise_type': 'laplace', 'n_samples': 100}\n", "{'totalnoise': 0.25, 'L_cov_21s': [-0.9, -0.7, -0.5, -0.3, 0, 0.3, 0.5, 0.7, 0.9], 'n_confs': 10, 'model_noise_type': 'gg', 'data_noise_type': 'uniform', 'n_samples': 100}\n", "{'totalnoise': 0.5, 'L_cov_21s': [-0.9, -0.7, -0.5, -0.3, 0, 0.3, 0.5, 0.7, 0.9], 'n_confs': 1, 'model_noise_type': 'gg', 'data_noise_type': 'laplace', 'n_samples': 100}\n", "{'totalnoise': 0.5, 'L_cov_21s': [-0.9, -0.7, -0.5, -0.3, 0, 0.3, 0.5, 0.7, 0.9], 'n_confs': 1, 'model_noise_type': 'gg', 'data_noise_type': 'uniform', 'n_samples': 100}\n", "{'totalnoise': 0.5, 'L_cov_21s': [-0.9, -0.7, -0.5, -0.3, 0, 0.3, 0.5, 0.7, 0.9], 'n_confs': 3, 'model_noise_type': 'gg', 'data_noise_type': 'laplace', 'n_samples': 100}\n", "{'totalnoise': 0.5, 'L_cov_21s': [-0.9, -0.7, -0.5, -0.3, 0, 0.3, 0.5, 0.7, 0.9], 'n_confs': 3, 'model_noise_type': 'gg', 'data_noise_type': 'uniform', 'n_samples': 100}\n", "{'totalnoise': 0.5, 'L_cov_21s': [-0.9, -0.7, -0.5, -0.3, 0, 0.3, 0.5, 0.7, 0.9], 'n_confs': 5, 'model_noise_type': 'gg', 'data_noise_type': 'laplace', 'n_samples': 100}\n", "{'totalnoise': 0.5, 'L_cov_21s': [-0.9, -0.7, -0.5, -0.3, 0, 0.3, 0.5, 0.7, 0.9], 'n_confs': 5, 'model_noise_type': 'gg', 'data_noise_type': 'uniform', 'n_samples': 100}\n", "{'totalnoise': 0.5, 'L_cov_21s': [-0.9, -0.7, -0.5, -0.3, 0, 0.3, 0.5, 0.7, 0.9], 'n_confs': 10, 'model_noise_type': 'gg', 'data_noise_type': 'laplace', 'n_samples': 100}\n", "{'totalnoise': 0.5, 'L_cov_21s': [-0.9, -0.7, -0.5, -0.3, 0, 0.3, 0.5, 0.7, 0.9], 'n_confs': 10, 'model_noise_type': 'gg', 'data_noise_type': 'uniform', 'n_samples': 100}\n", "{'totalnoise': 1.0, 'L_cov_21s': [-0.9, -0.7, -0.5, -0.3, 0, 0.3, 0.5, 0.7, 0.9], 'n_confs': 1, 'model_noise_type': 'gg', 'data_noise_type': 'laplace', 'n_samples': 100}\n", "{'totalnoise': 1.0, 'L_cov_21s': [-0.9, -0.7, -0.5, -0.3, 0, 0.3, 0.5, 0.7, 0.9], 'n_confs': 1, 'model_noise_type': 'gg', 'data_noise_type': 'uniform', 'n_samples': 100}\n", "{'totalnoise': 1.0, 'L_cov_21s': [-0.9, -0.7, -0.5, -0.3, 0, 0.3, 0.5, 0.7, 0.9], 'n_confs': 3, 'model_noise_type': 'gg', 'data_noise_type': 'laplace', 'n_samples': 100}\n", "{'totalnoise': 1.0, 'L_cov_21s': [-0.9, -0.7, -0.5, -0.3, 0, 0.3, 0.5, 0.7, 0.9], 'n_confs': 3, 'model_noise_type': 'gg', 'data_noise_type': 'uniform', 'n_samples': 100}\n", "{'totalnoise': 1.0, 'L_cov_21s': [-0.9, -0.7, -0.5, -0.3, 0, 0.3, 0.5, 0.7, 0.9], 'n_confs': 5, 'model_noise_type': 'gg', 'data_noise_type': 'laplace', 'n_samples': 100}\n", "{'totalnoise': 1.0, 'L_cov_21s': [-0.9, -0.7, -0.5, -0.3, 0, 0.3, 0.5, 0.7, 0.9], 'n_confs': 5, 'model_noise_type': 'gg', 'data_noise_type': 'uniform', 'n_samples': 100}\n", "{'totalnoise': 1.0, 'L_cov_21s': [-0.9, -0.7, -0.5, -0.3, 0, 0.3, 0.5, 0.7, 0.9], 'n_confs': 10, 'model_noise_type': 'gg', 'data_noise_type': 'laplace', 'n_samples': 100}\n", "{'totalnoise': 1.0, 'L_cov_21s': [-0.9, -0.7, -0.5, -0.3, 0, 0.3, 0.5, 0.7, 0.9], 'n_confs': 10, 'model_noise_type': 'gg', 'data_noise_type': 'uniform', 'n_samples': 100}\n", "{'totalnoise': 3.0, 'L_cov_21s': [-0.9, -0.7, -0.5, -0.3, 0, 0.3, 0.5, 0.7, 0.9], 'n_confs': 1, 'model_noise_type': 'gg', 'data_noise_type': 'laplace', 'n_samples': 100}\n", "{'totalnoise': 3.0, 'L_cov_21s': [-0.9, -0.7, -0.5, -0.3, 0, 0.3, 0.5, 0.7, 0.9], 'n_confs': 1, 'model_noise_type': 'gg', 'data_noise_type': 'uniform', 'n_samples': 100}\n", "{'totalnoise': 3.0, 'L_cov_21s': [-0.9, -0.7, -0.5, -0.3, 0, 0.3, 0.5, 0.7, 0.9], 'n_confs': 3, 'model_noise_type': 'gg', 'data_noise_type': 'laplace', 'n_samples': 100}\n", "{'totalnoise': 3.0, 'L_cov_21s': [-0.9, -0.7, -0.5, -0.3, 0, 0.3, 0.5, 0.7, 0.9], 'n_confs': 3, 'model_noise_type': 'gg', 'data_noise_type': 'uniform', 'n_samples': 100}\n", "{'totalnoise': 3.0, 'L_cov_21s': [-0.9, -0.7, -0.5, -0.3, 0, 0.3, 0.5, 0.7, 0.9], 'n_confs': 5, 'model_noise_type': 'gg', 'data_noise_type': 'laplace', 'n_samples': 100}\n", "{'totalnoise': 3.0, 'L_cov_21s': [-0.9, -0.7, -0.5, -0.3, 0, 0.3, 0.5, 0.7, 0.9], 'n_confs': 5, 'model_noise_type': 'gg', 'data_noise_type': 'uniform', 'n_samples': 100}\n", "{'totalnoise': 3.0, 'L_cov_21s': [-0.9, -0.7, -0.5, -0.3, 0, 0.3, 0.5, 0.7, 0.9], 'n_confs': 10, 'model_noise_type': 'gg', 'data_noise_type': 'laplace', 'n_samples': 100}\n", "{'totalnoise': 3.0, 'L_cov_21s': [-0.9, -0.7, -0.5, -0.3, 0, 0.3, 0.5, 0.7, 0.9], 'n_confs': 10, 'model_noise_type': 'gg', 'data_noise_type': 'uniform', 'n_samples': 100}\n", "Number of skipped trials = 1600\n" ] } ], "source": [ "df = run_expr(conds)" ] }, { "cell_type": "markdown", "metadata": { "collapsed": true }, "source": [ "## Result table" ] }, { "cell_type": "code", "execution_count": 8, "metadata": { "collapsed": false }, "outputs": [ { "data": { "text/html": [ "
\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
2log(bf)correct_rate
data noise typen_confstotalnoise
laplace10.2510.6585430.84
0.509.9568510.76
1.008.1325850.78
3.006.0675670.44
30.259.7018380.82
0.508.9658920.80
1.006.5923640.64
3.005.5858700.38
50.259.9736920.88
0.509.5252520.80
1.007.7317060.70
3.006.3416960.60
100.2510.6900530.86
0.5010.3494710.86
1.008.0077140.74
3.005.4911670.52
uniform10.2511.5573020.88
0.5010.5210020.76
1.007.9511350.68
3.005.2530980.48
30.2512.8159200.90
0.5011.6609200.86
1.007.9517250.74
3.006.0084830.56
50.2511.0165790.86
0.5010.2144590.78
1.006.8458860.66
3.005.5128820.52
100.2511.6832180.88
0.5010.1477680.82
1.007.3350320.62
3.006.2055800.50
\n", "
" ], "text/plain": [ " 2log(bf) correct_rate\n", "data noise type n_confs totalnoise \n", "laplace 1 0.25 10.658543 0.84\n", " 0.50 9.956851 0.76\n", " 1.00 8.132585 0.78\n", " 3.00 6.067567 0.44\n", " 3 0.25 9.701838 0.82\n", " 0.50 8.965892 0.80\n", " 1.00 6.592364 0.64\n", " 3.00 5.585870 0.38\n", " 5 0.25 9.973692 0.88\n", " 0.50 9.525252 0.80\n", " 1.00 7.731706 0.70\n", " 3.00 6.341696 0.60\n", " 10 0.25 10.690053 0.86\n", " 0.50 10.349471 0.86\n", " 1.00 8.007714 0.74\n", " 3.00 5.491167 0.52\n", "uniform 1 0.25 11.557302 0.88\n", " 0.50 10.521002 0.76\n", " 1.00 7.951135 0.68\n", " 3.00 5.253098 0.48\n", " 3 0.25 12.815920 0.90\n", " 0.50 11.660920 0.86\n", " 1.00 7.951725 0.74\n", " 3.00 6.008483 0.56\n", " 5 0.25 11.016579 0.86\n", " 0.50 10.214459 0.78\n", " 1.00 6.845886 0.66\n", " 3.00 5.512882 0.52\n", " 10 0.25 11.683218 0.88\n", " 0.50 10.147768 0.82\n", " 1.00 7.335032 0.62\n", " 3.00 6.205580 0.50" ] }, "execution_count": 8, "metadata": {}, "output_type": "execute_result" } ], "source": [ "import pandas as pd\n", "\n", "df_file = './20160822-eval-bml-results.pklz'\n", "df = load_pklz(df_file)\n", "df = pd.concat(\n", " {\n", " '2log(bf)': df['log_bf'], \n", " 'correct rate': df['correct_rate'], \n", " 'totalnoise': df['totalnoise'], \n", " 'data noise type': df['data_noise_type'], \n", " 'n_confs': df['n_confs']\n", " }, axis=1\n", ")\n", "sg = df.groupby(['data noise type', 'n_confs', 'totalnoise'])\n", "sg1 = sg['correct rate'].mean()\n", "sg2 = sg['2log(bf)'].mean()\n", "\n", "pd.concat(\n", " {\n", " 'correct_rate': sg1, \n", " '2log(bf)': sg2, \n", " }, axis=1\n", ")" ] }, { "cell_type": "markdown", "metadata": { "collapsed": true }, "source": [ "## Bayes factor and accuracy" ] }, { "cell_type": "code", "execution_count": 9, "metadata": { "collapsed": false }, "outputs": [ { "data": { "text/html": [ "
\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
countcorrect rate
data noise typelaplaceuniformlaplaceuniform
totalnoise2log(bf)
0.25(0, 2]21.018.00.6363640.642857
(2, 6]26.031.00.7222220.794872
(6, 10]37.020.00.8222220.869565
(10, 100]86.0107.01.0000000.972727
0.50(0, 2]13.016.00.4814810.533333
(2, 6]37.027.00.7115380.613636
(6, 10]37.024.00.8409090.800000
(10, 100]74.094.00.9610390.979167
1.00(0, 2]21.018.00.5675680.600000
(2, 6]36.039.00.6315790.520000
(6, 10]27.034.00.6000000.723404
(10, 100]59.044.00.9672130.916667
3.00(0, 2]28.030.00.5090910.600000
(2, 6]34.036.00.4657530.486486
(6, 10]17.020.00.5151520.500000
(10, 100]18.017.00.4615380.472222
\n", "
" ], "text/plain": [ " count correct rate \n", "data noise type laplace uniform laplace uniform\n", "totalnoise 2log(bf) \n", "0.25 (0, 2] 21.0 18.0 0.636364 0.642857\n", " (2, 6] 26.0 31.0 0.722222 0.794872\n", " (6, 10] 37.0 20.0 0.822222 0.869565\n", " (10, 100] 86.0 107.0 1.000000 0.972727\n", "0.50 (0, 2] 13.0 16.0 0.481481 0.533333\n", " (2, 6] 37.0 27.0 0.711538 0.613636\n", " (6, 10] 37.0 24.0 0.840909 0.800000\n", " (10, 100] 74.0 94.0 0.961039 0.979167\n", "1.00 (0, 2] 21.0 18.0 0.567568 0.600000\n", " (2, 6] 36.0 39.0 0.631579 0.520000\n", " (6, 10] 27.0 34.0 0.600000 0.723404\n", " (10, 100] 59.0 44.0 0.967213 0.916667\n", "3.00 (0, 2] 28.0 30.0 0.509091 0.600000\n", " (2, 6] 34.0 36.0 0.465753 0.486486\n", " (6, 10] 17.0 20.0 0.515152 0.500000\n", " (10, 100] 18.0 17.0 0.461538 0.472222" ] }, "execution_count": 9, "metadata": {}, "output_type": "execute_result" } ], "source": [ "import pandas as pd\n", "\n", "def count(x): return np.sum(x.astype(int))\n", "data_dir = '.'\n", "df_file = data_dir + '/20160822-eval-bml-results.pklz'\n", "df = load_pklz(df_file)\n", "\n", "df = pd.concat(\n", " {\n", " '2log(bf)': df['log_bf'], \n", " 'correct rate': df['correct_rate'], \n", " 'count': df['correct_rate'], \n", " 'totalnoise': df['totalnoise'], \n", " 'data noise type': df['data_noise_type']\n", " }, axis=1\n", ")\n", "\n", "df = df.pivot_table(values=['correct rate', 'count'], \n", " index=['totalnoise', pd.cut(df['2log(bf)'], [0., 2., 6., 10., 100.])], \n", " columns='data noise type', \n", " aggfunc={'correct rate': np.mean, 'count': np.sum})\n", "df" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": true }, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.5.2" }, "nav_menu": {}, "toc": { "navigate_menu": true, "number_sections": true, "sideBar": true, "threshold": 6, "toc_cell": false, "toc_section_display": "block", "toc_window_display": false }, "toc_position": { "height": "890px", "left": "0px", "right": "1414px", "top": "106px", "width": "359px" } }, "nbformat": 4, "nbformat_minor": 0 }