{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 144,
   "outputs": [],
   "source": [
    "from pymongo import MongoClient\n",
    "from matplotlib import pyplot as plt\n",
    "import numpy as np\n",
    "from dcase2020_task2.data_sets.mcm_dataset import INVERSE_CLASS_MAP, TRAINING_ID_MAP, EVALUATION_ID_MAP, CLASS_MAP\n",
    "from scipy.stats import rankdata\n",
    "import os\n",
    "from shutil import copyfile\n",
    "import pandas as pd\n",
    "\n",
    "baseline_auc = {\n",
    "    'name': 'baseline',\n",
    "    0: {0: 0.5441, 2: 0.7340, 4: 0.6161, 6: 0.7392},\n",
    "    1: {0: 0.6715, 2: 0.6153, 4: 0.8833, 6: 0.7455},\n",
    "    2: {0: 0.9619, 2: 0.7897, 4: 0.9430, 6: 0.6959},\n",
    "    3: {1: 0.8136, 2: 0.8597, 3: 0.6330, 4: 0.8445},\n",
    "    4: {1: 0.7807, 2: 0.6416, 3: 0.7535},\n",
    "    5: {0: 0.6876, 2: 0.6818, 4: 0.7430, 6: 0.5390}\n",
    "    }\n",
    "    \n",
    "baseline_pauc = {\n",
    "    'name': 'baseline',\n",
    "    0: {0: 0.4937, 2: 0.5481, 4: 0.5326, 6: 0.5235},\n",
    "    1: {0: 0.5674, 2: 0.5810, 4: 0.6710, 6: 0.5802},\n",
    "    2: {0: 0.8144, 2: 0.6368, 4: 0.7198, 6: 0.4902},\n",
    "    3: {1: 0.6840, 2: 0.7772, 3: 0.5521, 4: 0.6897},\n",
    "    4: {1: 0.6425, 2: 0.5601, 3: 0.6103},\n",
    "    5: {0: 0.5170, 2: 0.5183, 4: 0.5197, 6: 0.4843}\n",
    "}\n",
    "\n",
    "baseline_both = {}\n",
    "for t in baseline_auc:\n",
    "    if t == 'name':\n",
    "        baseline_both[t] = 'baseline'\n",
    "        continue\n",
    "    else:\n",
    "        baseline_both[t] = {}\n",
    "    for i in baseline_auc[t]:\n",
    "        baseline_both[t][i] = np.array([baseline_auc[t][i], baseline_pauc[t][i]])\n",
    "\n",
    "\n",
    "def get_experiment(runs, name):\n",
    "    experiment_dict = dict()\n",
    "    for i in range(6):\n",
    "        experiment_dict[i] = dict()\n",
    "    \n",
    "    experiment_dict['name'] = name\n",
    "    \n",
    "    for experiment in runs:\n",
    "        if experiment['config'].get('id') == name:\n",
    "            machine_dict = experiment_dict.get(experiment['config']['machine_type'])\n",
    "            result = experiment.get('result')\n",
    "            machine_type = INVERSE_CLASS_MAP[experiment['config']['machine_type']]\n",
    "            machine_id = experiment['config']['machine_id']\n",
    "            \n",
    "            if result:\n",
    "                machine_dict[experiment['config']['machine_id']] = result.get(\n",
    "                    machine_type, {}\n",
    "                ).get(\n",
    "                    f'json://{machine_id}', -1\n",
    "                ).get('py/tuple', [0, 0])[:2]\n",
    "            else:\n",
    "                machine_dict[experiment['config']['machine_id']] = np.array([0, 0])\n",
    "    return experiment_dict\n",
    "\n",
    "\n",
    "def get_record(experiment):\n",
    "    record = []\n",
    "    for i in range(6):\n",
    "        for j in TRAINING_ID_MAP[i]:\n",
    "                v = experiment.get(i)\n",
    "                if v:\n",
    "                    v = v.get(j, [0, 0])\n",
    "                else:\n",
    "                    v = np.array([0, 0])\n",
    "                record.append(np.array(v))\n",
    "    assert len(record) == 23\n",
    "    return  experiment['name'], record"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "outputs": [
    {
     "name": "stdout",
     "text": [
      "Loaded 572 runs.\n"
     ],
     "output_type": "stream"
    }
   ],
   "source": [
    "client = MongoClient('mongodb://student2.cp.jku.at:27017/')\n",
    "experiments = [r for r in client.resnet_gridsearch.runs.find({\"experiment.name\": \"dcase2020_task2_ClassificationExperiment\"})]\n",
    "print(f'Loaded {len(experiments)} runs.')"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "outputs": [
    {
     "name": "stdout",
     "text": [
      "Loaded 25 distinct experiments.\n"
     ],
     "output_type": "stream"
    }
   ],
   "source": [
    "descriptors = set()\n",
    "for experiment in experiments:\n",
    "    descriptors = descriptors.union(set([experiment['config']['id']]))\n",
    "descriptors = list(descriptors)\n",
    "print(f'Loaded {len(descriptors)} distinct experiments.')"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "outputs": [
    {
     "name": "stdout",
     "text": [
      "Loaded 13 distinct experiments, without reruns.\n"
     ],
     "output_type": "stream"
    }
   ],
   "source": [
    "descriptors = [d for d in descriptors if d.split('_')[-1] != 'rerun']\n",
    "# descriptors = [d for d in descriptors if d.split('_')[2] != '2']\n",
    "# for descriptor in descriptors:\n",
    "#     print(descriptor)\n",
    "    \n",
    "print(f'Loaded {len(descriptors)} distinct experiments, without reruns.')"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "outputs": [],
   "source": [
    "# Extract Results\n",
    "# Concatenate Baseline Results\n",
    "n, m = get_record(baseline_both)\n",
    "names = [n]\n",
    "metrics = [np.array(m)]\n",
    "\n",
    "for descriptor in descriptors:\n",
    "    n, m = get_record(\n",
    "            get_experiment(\n",
    "                experiments, \n",
    "                descriptor\n",
    "            )\n",
    "        )\n",
    "    names.append(n)\n",
    "    metrics.append(np.array(m))"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "outputs": [
    {
     "name": "stdout",
     "text": [
      "Best Model for Machine Type 0: [ 8  2 10  1  5  6 11  4  9  7 13 12  3  0]\n",
      "Best Model for Machine Type 1: [ 8  4  2  6  9 10 13 11  5  1  7  3 12  0]\n",
      "Best Model for Machine Type 2: [ 4 10  1  8  5  9 11 13  6  2 12  3  7  0]\n",
      "Best Model for Machine Type 3: [11  1 12 10  5  2  4  9  8 13  6  3  7  0]\n",
      "Best Model for Machine Type 4: [13  9  1  5  4  6  7  2  3 12 10 11  8  0]\n",
      "Best Model for Machine Type 5: [ 8 12  3 11  7  4 10  6  2 13  5  1  9  0]\n",
      "00: ID-04 resnet_gridsearch_a_bit_larger_loose_1e-4_100_BCE\n",
      "01: ID-08 resnet_gridsearch_2_a_bit_larger_loose_1e-4_0.99_100_BCE\n",
      "02: ID-10 resnet_gridsearch_a_bit_larger_loose_1e-4_100_AUC\n",
      "03: ID-01 resnet_gridsearch_a_bit_larger_loose_1e-5_100_BCE\n",
      "04: ID-11 resnet_gridsearch_a_bit_smaller_loose_1e-4_100_BCE\n",
      "05: ID-05 resnet_gridsearch_a_bit_larger_loose_1e-5_100_AUC\n",
      "06: ID-02 resnet_gridsearch_normal_loose_1e-4_100_BCE\n",
      "07: ID-09 resnet_gridsearch_normal_loose_1e-5_100_BCE\n",
      "08: ID-06 resnet_gridsearch_normal_loose_1e-4_100_AUC\n",
      "09: ID-12 resnet_gridsearch_a_bit_smaller_loose_1e-4_100_AUC\n",
      "10: ID-13 resnet_gridsearch_normal_loose_1e-5_100_AUC\n",
      "11: ID-07 resnet_gridsearch_a_bit_smaller_loose_1e-5_100_AUC\n",
      "12: ID-03 resnet_gridsearch_a_bit_smaller_loose_1e-5_100_BCE\n",
      "13: ID-00 baseline\n"
     ],
     "output_type": "stream"
    }
   ],
   "source": [
    "data = np.array(metrics)\n",
    "auc_ranks = []\n",
    "pauc_ranks = []\n",
    "idxes = [0, 4, 8, 12, 16, 19, 23]\n",
    "best_idxes = []\n",
    "for type_, (i, j) in enumerate(zip(idxes[:-1], idxes[1:])):\n",
    "    average_auc = data[:, i:j, 0].mean(axis=1)\n",
    "    average_pauc = data[:, i:j, 1].mean(axis=1)\n",
    "    best_idxes.append(\n",
    "       np.argsort(average_auc + average_pauc)[::-1]\n",
    "    )\n",
    "    print(f'Best Model for Machine Type {type_}: {best_idxes[-1]}')\n",
    "    auc_ranks.append(rankdata(-average_auc))\n",
    "    pauc_ranks.append(rankdata(-average_pauc))\n",
    "\n",
    "\n",
    "ranks = np.stack([np.array(list(zip(*auc_ranks))), np.array(list(zip(*pauc_ranks)))], axis=-1).mean(axis=-1).mean(axis=-1)\n",
    "\n",
    "sorted_model_indices = list(np.argsort(ranks))\n",
    "names = np.array(names)\n",
    "for i, (n, r, j) in enumerate(zip(names[sorted_model_indices], ranks[sorted_model_indices], sorted_model_indices)):\n",
    "    print(f'{i:02d}: ID-{j:02d} {n}')"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 156,
   "outputs": [],
   "source": [
    "import sklearn\n",
    "\n",
    "def compute_auc(src):\n",
    "    scores = pd.read_csv(src, names=['file_name', 'score'], index_col=False).to_numpy()[:, 1]\n",
    "    names = pd.read_csv(src, names=['file_name', 'score'], index_col=False).to_numpy()[:, 0]\n",
    "    names = np.array([1 if name.split('_')[0] == 'anomaly' else 0 for name in names])\n",
    "    return sklearn.metrics.roc_auc_score(names, scores), sklearn.metrics.roc_auc_score(names, scores, max_fpr=0.1)\n",
    "\n",
    "run_ids = names\n"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "outputs": [],
   "source": [
    "# Create Submission 1"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 158,
   "outputs": [],
   "source": [
    "for machine_type in range(6):\n",
    "    for machine_id in EVALUATION_ID_MAP[machine_type]:\n",
    "        best_model_folder = run_ids[sorted_model_indices[0]]\n",
    "        src_path = os.path.join('..', 'experiment_logs', best_model_folder)\n",
    "        src = os.path.join(src_path, f'anomaly_score_{INVERSE_CLASS_MAP[machine_type]}_id_{machine_id}_mean.csv')\n",
    "        dst_path = os.path.join('..', 'submission_package', 'task2', 'Primus_CP-JKU_task2_1')\n",
    "        dst = os.path.join(dst_path, f'anomaly_score_{INVERSE_CLASS_MAP[machine_type]}_id_{machine_id:02d}.csv')\n",
    "        copyfile(src, dst)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 159,
   "outputs": [
    {
     "name": "stdout",
     "text": [
      "\tfan:\n",
      "\t\taveraged_auc: 0.9226685446121019\n",
      "\t\taveraged_pauc: 0.8230212986543588\n",
      "\tpump:\n",
      "\t\taveraged_auc: 0.9297781495399142\n",
      "\t\taveraged_pauc: 0.8722867745313565\n",
      "\tslider:\n",
      "\t\taveraged_auc: 0.9894779962546816\n",
      "\t\taveraged_pauc: 0.9454464813719693\n",
      "\tToyCar:\n",
      "\t\taveraged_auc: 0.9489897900841298\n",
      "\t\taveraged_pauc: 0.8752479934828495\n",
      "\tToyConveyor:\n",
      "\t\taveraged_auc: 0.83764533636162\n",
      "\t\taveraged_pauc: 0.7279798253268154\n",
      "\tvalve:\n",
      "\t\taveraged_auc: 0.9424583333333334\n",
      "\t\taveraged_pauc: 0.8903508771929824\n"
     ],
     "output_type": "stream"
    }
   ],
   "source": [
    "for machine_type in range(6):\n",
    "    auc = []\n",
    "    pauc = []\n",
    "    for machine_id in TRAINING_ID_MAP[machine_type]:\n",
    "        best_model_folder = run_ids[sorted_model_indices[0]]\n",
    "        src_path = os.path.join('..', 'experiment_logs', best_model_folder)\n",
    "        src = os.path.join(src_path, f'anomaly_score_{INVERSE_CLASS_MAP[machine_type]}_id_{machine_id}_mean.csv')\n",
    "        a, p = compute_auc(src)\n",
    "        auc.append(a)\n",
    "        pauc.append(p)\n",
    "        \n",
    "    print(f'\\t{INVERSE_CLASS_MAP[machine_type]}:\\n\\t\\taveraged_auc: {np.mean(auc)}\\n\\t\\taveraged_pauc: {np.mean(pauc)}')"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 160,
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "# Create Submission 2"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% \n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 161,
   "outputs": [],
   "source": [
    "for machine_type, idxes in enumerate(best_idxes):\n",
    "    for machine_id in EVALUATION_ID_MAP[machine_type]:\n",
    "        \n",
    "        idx = idxes[0]\n",
    "        best_model_folder = run_ids[idx]\n",
    "\n",
    "        src_path = os.path.join('..', 'experiment_logs', best_model_folder)\n",
    "        src = os.path.join(src_path, f'anomaly_score_{INVERSE_CLASS_MAP[machine_type]}_id_{machine_id}_mean.csv')\n",
    "        dst_path = os.path.join('..', 'submission_package', 'task2', 'Primus_CP-JKU_task2_2')\n",
    "        dst = os.path.join(dst_path, f'anomaly_score_{INVERSE_CLASS_MAP[machine_type]}_id_{machine_id:02d}.csv')\n",
    "        copyfile(src, dst)\n",
    "        "
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "outputs": [
    {
     "name": "stdout",
     "text": [
      "\tfan:\n",
      "\t\taveraged_auc: 0.9286317167841518\n",
      "\t\taveraged_pauc: 0.8352913487070679\n",
      "\tpump:\n",
      "\t\taveraged_auc: 0.9297781495399142\n",
      "\t\taveraged_pauc: 0.8722867745313565\n",
      "\tslider:\n",
      "\t\taveraged_auc: 0.9894779962546816\n",
      "\t\taveraged_pauc: 0.9454464813719693\n",
      "\tToyCar:\n",
      "\t\taveraged_auc: 0.9566950093931226\n",
      "\t\taveraged_pauc: 0.8961968600747151\n",
      "\tToyConveyor:\n",
      "\t\taveraged_auc: 0.8526503235962499\n",
      "\t\taveraged_pauc: 0.7259891865658302\n",
      "\tvalve:\n",
      "\t\taveraged_auc: 0.9776656162464985\n",
      "\t\taveraged_pauc: 0.9357400855078873\n"
     ],
     "output_type": "stream"
    }
   ],
   "source": [
    "for machine_type, idxes in enumerate(best_idxes):\n",
    "    auc = []\n",
    "    pauc = []\n",
    "    for machine_id in TRAINING_ID_MAP[machine_type]:\n",
    "        idx = idxes[0]\n",
    "        best_model_folder = run_ids[idx]\n",
    "        src_path = os.path.join('..', 'experiment_logs', best_model_folder)\n",
    "        src = os.path.join(src_path, f'anomaly_score_{INVERSE_CLASS_MAP[machine_type]}_id_{machine_id}_mean.csv')\n",
    "        a, p = compute_auc(src)\n",
    "        auc.append(a)\n",
    "        pauc.append(p)\n",
    "        \n",
    "    print(f'\\t{INVERSE_CLASS_MAP[machine_type]}:\\n\\t\\taveraged_auc: {np.mean(auc)}\\n\\t\\taveraged_pauc: {np.mean(pauc)}')"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 162,
   "outputs": [],
   "source": [
    "# Create Submission 3 # median ensemble"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 168,
   "outputs": [],
   "source": [
    "for machine_type, idxes in enumerate(best_idxes):\n",
    "    for machine_id in EVALUATION_ID_MAP[machine_type]:\n",
    "        file_names = []\n",
    "        scores = []\n",
    "        for idx in idxes[:5]:\n",
    "            best_model_folder = run_ids[idx]\n",
    "            src_path = os.path.join('..', 'experiment_logs', best_model_folder)\n",
    "            src = os.path.join(src_path, f'anomaly_score_{INVERSE_CLASS_MAP[machine_type]}_id_{machine_id}_mean.csv')\n",
    "            scores.append(pd.read_csv(src, names=['file_name', 'score'], index_col=False).to_numpy()[:, 1])\n",
    "            file_names.append(pd.read_csv(src, names=['file_name', 'score'], index_col=False).to_numpy()[:, 0])\n",
    "        \n",
    "        scores = list(np.median(np.array(scores).T, axis=-1).reshape(-1))\n",
    "        dst_path = os.path.join('..', 'submission_package', 'task2', 'Primus_CP-JKU_task2_3')\n",
    "        dst = os.path.join(dst_path, f'anomaly_score_{INVERSE_CLASS_MAP[machine_type]}_id_{machine_id:02d}.csv')\n",
    "\n",
    "        pd.DataFrame(list(zip(file_names[0], scores))).to_csv(dst, index=False, header=False)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 169,
   "outputs": [
    {
     "name": "stdout",
     "text": [
      "\tfan:\n",
      "\t\taveraged_auc: 0.9281888985937587\n",
      "\t\taveraged_pauc: 0.8283523606556178\n",
      "\tpump:\n",
      "\t\taveraged_auc: 0.9209936334730452\n",
      "\t\taveraged_pauc: 0.8705862272890137\n",
      "\tslider:\n",
      "\t\taveraged_auc: 0.9858871722846442\n",
      "\t\taveraged_pauc: 0.9268061304947763\n",
      "\tToyCar:\n",
      "\t\taveraged_auc: 0.9547106714040676\n",
      "\t\taveraged_pauc: 0.8913650442572985\n",
      "\tToyConveyor:\n",
      "\t\taveraged_auc: 0.8514805262648587\n",
      "\t\taveraged_pauc: 0.7374989794507775\n",
      "\tvalve:\n",
      "\t\taveraged_auc: 0.9696039915966386\n",
      "\t\taveraged_pauc: 0.9118411838419578\n"
     ],
     "output_type": "stream"
    }
   ],
   "source": [
    "\n",
    "for machine_type, idxes in enumerate(best_idxes):\n",
    "    auc = []\n",
    "    pauc = []\n",
    "    for machine_id in TRAINING_ID_MAP[machine_type]:\n",
    "        file_names = []\n",
    "        scores = []\n",
    "        for idx in idxes[:5]:\n",
    "            best_model_folder = run_ids[idx]\n",
    "            src_path = os.path.join('..', 'experiment_logs', best_model_folder)\n",
    "            src = os.path.join(src_path, f'anomaly_score_{INVERSE_CLASS_MAP[machine_type]}_id_{machine_id}_mean.csv')\n",
    "            scores.append(pd.read_csv(src, names=['file_name', 'score'], index_col=False).to_numpy()[:, 1])\n",
    "            file_names.append(pd.read_csv(src, names=['file_name', 'score'], index_col=False).to_numpy()[:, 0])\n",
    "        \n",
    "        scores = list(np.median(np.array(scores).T, axis=-1).reshape(-1))\n",
    "        file_names = np.array([1 if name.split('_')[0] == 'anomaly' else 0 for name in file_names[0]])      \n",
    "        a, p = sklearn.metrics.roc_auc_score(file_names, scores), sklearn.metrics.roc_auc_score(file_names, scores, max_fpr=0.1)\n",
    "        auc.append(a)\n",
    "        pauc.append(p)\n",
    "            \n",
    "    print(f'\\t{INVERSE_CLASS_MAP[machine_type]}:\\n\\t\\taveraged_auc: {np.mean(auc)}\\n\\t\\taveraged_pauc: {np.mean(pauc)}')"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 165,
   "outputs": [],
   "source": [
    "# Create Submission 4 # mean ensemble"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 175,
   "outputs": [],
   "source": [
    "for machine_type, idxes in enumerate(best_idxes):\n",
    "    for machine_id in EVALUATION_ID_MAP[machine_type]:\n",
    "        file_names = []\n",
    "        scores = []\n",
    "        for idx in idxes[:13]:\n",
    "            best_model_folder = run_ids[idx]\n",
    "            src_path = os.path.join('..', 'experiment_logs', best_model_folder)\n",
    "            src = os.path.join(src_path, f'anomaly_score_{INVERSE_CLASS_MAP[machine_type]}_id_{machine_id}_mean.csv')\n",
    "            scores.append(pd.read_csv(src, names=['file_name', 'score'], index_col=False).to_numpy()[:, 1])\n",
    "            file_names.append(pd.read_csv(src, names=['file_name', 'score'], index_col=False).to_numpy()[:, 0])\n",
    "        \n",
    "        scores = list(np.median(np.array(scores).T, axis=-1).reshape(-1))\n",
    "        dst_path = os.path.join('..', 'submission_package', 'task2', 'Primus_CP-JKU_task2_4')\n",
    "        dst = os.path.join(dst_path, f'anomaly_score_{INVERSE_CLASS_MAP[machine_type]}_id_{machine_id:02d}.csv')\n",
    "\n",
    "        pd.DataFrame(list(zip(file_names[0], scores))).to_csv(dst, index=False, header=False)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 176,
   "outputs": [
    {
     "name": "stdout",
     "text": [
      "\tfan:\n",
      "\t\taveraged_auc: 0.9230309586203593\n",
      "\t\taveraged_pauc: 0.8285210664235381\n",
      "\tpump:\n",
      "\t\taveraged_auc: 0.9146627457465693\n",
      "\t\taveraged_pauc: 0.8678065821022478\n",
      "\tslider:\n",
      "\t\taveraged_auc: 0.9822893258426966\n",
      "\t\taveraged_pauc: 0.9108762073723635\n",
      "\tToyCar:\n",
      "\t\taveraged_auc: 0.9504648370497427\n",
      "\t\taveraged_pauc: 0.8890240180210388\n",
      "\tToyConveyor:\n",
      "\t\taveraged_auc: 0.8254031447576784\n",
      "\t\taveraged_pauc: 0.7026976317730064\n",
      "\tvalve:\n",
      "\t\taveraged_auc: 0.93825\n",
      "\t\taveraged_pauc: 0.8792763157894736\n"
     ],
     "output_type": "stream"
    }
   ],
   "source": [
    "\n",
    "for machine_type, idxes in enumerate(best_idxes):\n",
    "    auc = []\n",
    "    pauc = []\n",
    "    for machine_id in TRAINING_ID_MAP[machine_type]:\n",
    "        file_names = []\n",
    "        scores = []\n",
    "        for idx in idxes[:13]:\n",
    "            best_model_folder = run_ids[idx]\n",
    "            src_path = os.path.join('..', 'experiment_logs', best_model_folder)\n",
    "            src = os.path.join(src_path, f'anomaly_score_{INVERSE_CLASS_MAP[machine_type]}_id_{machine_id}_mean.csv')\n",
    "            scores.append(pd.read_csv(src, names=['file_name', 'score'], index_col=False).to_numpy()[:, 1])\n",
    "            file_names.append(pd.read_csv(src, names=['file_name', 'score'], index_col=False).to_numpy()[:, 0])\n",
    "        \n",
    "        scores = list(np.median(np.array(scores).T, axis=-1).reshape(-1))\n",
    "        file_names = np.array([1 if name.split('_')[0] == 'anomaly' else 0 for name in file_names[0]])      \n",
    "        a, p = sklearn.metrics.roc_auc_score(file_names, scores), sklearn.metrics.roc_auc_score(file_names, scores, max_fpr=0.1)\n",
    "        auc.append(a)\n",
    "        pauc.append(p)\n",
    "            \n",
    "    print(f'\\t{INVERSE_CLASS_MAP[machine_type]}:\\n\\t\\taveraged_auc: {np.mean(auc)}\\n\\t\\taveraged_pauc: {np.mean(pauc)}')"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "\n"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  }
 ],
 "metadata": {
  "kernelspec": {
   "name": "python3",
   "language": "python",
   "display_name": "Python 3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  },
  "pycharm": {
   "stem_cell": {
    "cell_type": "raw",
    "source": [],
    "metadata": {
     "collapsed": false
    }
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}