{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true,
    "pycharm": {
     "is_executing": false
    }
   },
   "outputs": [],
   "source": [
    "import dcase2020_task2\n",
    "\n",
    "from pymongo import MongoClient\n",
    "import numpy as np\n",
    "\n",
    "INVERSE_CLASS_MAP = {\n",
    "    0: 'fan',\n",
    "    1: 'pump',\n",
    "    2: 'slider',\n",
    "    3: 'ToyCar',\n",
    "    4: 'ToyConveyor',\n",
    "    5: 'valve'\n",
    "}\n",
    "\n",
    "baseline_auc = {\n",
    "    0: {0: 0.5441, 2: 0.7340, 4: 0.6161, 6: 0.7392},\n",
    "    1: {0: 0.6715, 2: 0.6153, 4: 0.8833, 6: 0.7455},\n",
    "    2: {0: 0.9619, 2: 0.7897, 4: 0.9430, 6: 0.6959},\n",
    "    3: {1: 0.8136, 2: 0.8597, 3: 0.6330, 4: 0.8445},\n",
    "    4: {1: 0.7807, 2: 0.6416, 3: 0.7535},\n",
    "    5: {0: 0.6876, 2: 0.6818, 4: 0.7430, 6: 0.5390}\n",
    "}\n",
    "\n",
    "baseline_pauc = {\n",
    "    0: {0: 0.4937, 2: 0.5481, 4: 0.5326, 6: 0.5235},\n",
    "    1: {0: 0.5674, 2: 0.5810, 4: 0.6710, 6: 0.5802},\n",
    "    2: {0: 0.8144, 2: 0.6368, 4: 0.7198, 6: 0.4902},\n",
    "    3: {1: 0.6840, 2: 0.7772, 3: 0.5521, 4: 0.6897},\n",
    "    4: {1: 0.6425, 2: 0.5601, 3: 0.6103},\n",
    "    5: {0: 0.5170, 2: 0.5183, 4: 0.5197, 6: 0.4843}\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "pycharm": {
     "is_executing": false,
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['fs.chunks',\n",
       " 'omniboard.settings',\n",
       " 'omniboard.metric.columns',\n",
       " 'omniboard.custom.columns',\n",
       " 'fs.files',\n",
       " 'runs',\n",
       " 'metrics']"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "client = MongoClient('mongodb://student2.cp.jku.at:27017/')\n",
    "db = client.dcase2020_task2\n",
    "db.list_collection_names()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "pycharm": {
     "is_executing": false,
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "20 experiments loaded\n",
      "| Machine | Type | ID   | AUC        | to BL      | AUC        | to BL      |\n",
      "| ------- | :--- | :--- | ---------- | ---------- | ---------- | ---------- |\n",
      "|fan\t| 0\t|0\t|0.5857\t|+0.0416\t|0.5846\t|+0.0909\t|\n",
      "|fan\t| 0\t|2\t|0.9865\t|+0.2525\t|0.9315\t|+0.3834\t|\n",
      "|fan\t| 0\t|4\t|0.7753\t|+0.1592\t|0.6568\t|+0.1242\t|\n",
      "|fan\t| 0\t|6\t|0.7520\t|+0.0128\t|0.8455\t|+0.3220\t|\n",
      "|pump\t| 1\t|0\t|0.8373\t|+0.1658\t|0.7126\t|+0.1452\t|\n",
      "|pump\t| 1\t|2\t|0.5577\t|-0.0576\t|0.5225\t|-0.0585\t|\n",
      "|pump\t| 1\t|4\t|0.9995\t|+0.1162\t|0.9974\t|+0.3264\t|\n",
      "|pump\t| 1\t|6\t|0.9311\t|+0.1856\t|0.8354\t|+0.2552\t|\n",
      "|slider\t| 2\t|0\t|0.9946\t|+0.0327\t|0.9821\t|+0.1677\t|\n",
      "|slider\t| 2\t|2\t|0.7940\t|+0.0043\t|0.5494\t|-0.0874\t|\n",
      "|slider\t| 2\t|4\t|0.9953\t|+0.0523\t|0.9752\t|+0.2554\t|\n",
      "|slider\t| 2\t|6\t|0.8202\t|+0.1243\t|0.5795\t|+0.0893\t|\n",
      "|ToyCar\t| 3\t|1\t|0.6782\t|-0.1354\t|0.6485\t|-0.0355\t|\n",
      "|ToyCar\t| 3\t|2\t|0.8749\t|+0.0152\t|0.7847\t|+0.0075\t|\n",
      "|ToyCar\t| 3\t|3\t|0.9746\t|+0.3416\t|0.9001\t|+0.3480\t|\n",
      "|ToyCar\t| 3\t|4\t|0.0000\t|-0.8445\t|0.0000\t|-0.6897\t|\n",
      "|ToyCar\t| 3\t|4\t|0.9975\t|+0.1530\t|0.9867\t|+0.2970\t|\n",
      "|ToyConveyor\t| 4\t|1\t|0.8521\t|+0.0714\t|0.7509\t|+0.1084\t|\n",
      "|ToyConveyor\t| 4\t|2\t|0.5919\t|-0.0497\t|0.5407\t|-0.0194\t|\n",
      "|ToyConveyor\t| 4\t|3\t|0.0000\t|-0.7535\t|0.0000\t|-0.6103\t|\n"
     ]
    }
   ],
   "source": [
    "experiments = []\n",
    "for experiment in db.runs.find(\n",
    "        {\n",
    "         \"experiment.name\": \"dcase2020_task2_classification\"\n",
    "        }\n",
    "):\n",
    "    if experiment['config'].get('descriptor') == 'not_normalized':\n",
    "        experiments.append([\n",
    "            experiment['config']['machine_type'], \n",
    "            experiment['config']['machine_id'], \n",
    "            experiment['result']['auroc_mean'] if experiment['result'] else 0,\n",
    "            experiment['result']['pauroc_mean'] if experiment['result'] else 0,\n",
    "        ])\n",
    "print(\"{} experiments loaded\".format(len(experiments)))  \n",
    "\n",
    "# assert len(experiments) == 23\n",
    "\n",
    "print(\"| Machine | Type | ID   | AUC        | to BL      | AUC        | to BL      |\")\n",
    "print(\"| ------- | :--- | :--- | ---------- | ---------- | ---------- | ---------- |\")\n",
    "for e in experiments:\n",
    "        print(\"|{}\\t| {}\\t|{}\\t|{:0.4f}\\t|{:+0.4f}\\t|{:0.4f}\\t|{:+1.4f}\\t|\".format(\n",
    "            INVERSE_CLASS_MAP[e[0]],\n",
    "            e[0],\n",
    "            e[1],\n",
    "            e[2],\n",
    "            e[2] - baseline_auc[e[0]][e[1]],\n",
    "            e[3],\n",
    "            e[3] - baseline_pauc[e[0]][e[1]]\n",
    "        ))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "pycharm": {
     "is_executing": false,
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "22 experiments loaded\n",
      "| Machine | Type | ID   | AUC        | to BL      | AUC        | to BL      |\n",
      "| ------- | :--- | :--- | ---------- | ---------- | ---------- | ---------- |\n",
      "|fan\t| 0\t|0\t|0.6031\t|+0.0590\t|0.5327\t|+0.0390\t|\n",
      "|fan\t| 0\t|2\t|0.9679\t|+0.2339\t|0.8651\t|+0.3170\t|\n",
      "|fan\t| 0\t|4\t|0.7782\t|+0.1621\t|0.6996\t|+0.1670\t|\n",
      "|fan\t| 0\t|6\t|0.7735\t|+0.0343\t|0.8456\t|+0.3221\t|\n",
      "|pump\t| 1\t|0\t|0.8815\t|+0.2100\t|0.7755\t|+0.2081\t|\n",
      "|pump\t| 1\t|2\t|0.5438\t|-0.0715\t|0.5268\t|-0.0542\t|\n",
      "|pump\t| 1\t|4\t|0.9999\t|+0.1166\t|0.9995\t|+0.3285\t|\n",
      "|pump\t| 1\t|6\t|0.9278\t|+0.1823\t|0.7038\t|+0.1236\t|\n",
      "|slider\t| 2\t|0\t|0.9807\t|+0.0188\t|0.9253\t|+0.1109\t|\n",
      "|slider\t| 2\t|2\t|0.8460\t|+0.0563\t|0.6611\t|+0.0243\t|\n",
      "|slider\t| 2\t|4\t|0.9522\t|+0.0092\t|0.8217\t|+0.1019\t|\n",
      "|slider\t| 2\t|6\t|0.8180\t|+0.1221\t|0.5742\t|+0.0840\t|\n",
      "|ToyCar\t| 3\t|1\t|0.6264\t|-0.1872\t|0.6610\t|-0.0230\t|\n",
      "|ToyCar\t| 3\t|2\t|0.7937\t|-0.0660\t|0.6393\t|-0.1379\t|\n",
      "|ToyCar\t| 3\t|3\t|0.8065\t|+0.1735\t|0.7319\t|+0.1798\t|\n",
      "|ToyCar\t| 3\t|4\t|0.9409\t|+0.0964\t|0.8307\t|+0.1410\t|\n",
      "|ToyConveyor\t| 4\t|1\t|0.8453\t|+0.0646\t|0.7436\t|+0.1011\t|\n",
      "|ToyConveyor\t| 4\t|2\t|0.5959\t|-0.0457\t|0.5218\t|-0.0383\t|\n",
      "|ToyConveyor\t| 4\t|3\t|0.6888\t|-0.0647\t|0.5803\t|-0.0300\t|\n",
      "|valve\t| 5\t|0\t|0.9460\t|+0.2584\t|0.7771\t|+0.2601\t|\n",
      "|valve\t| 5\t|2\t|0.9258\t|+0.2440\t|0.7482\t|+0.2299\t|\n",
      "|valve\t| 5\t|4\t|0.0000\t|-0.7430\t|0.0000\t|-0.5197\t|\n"
     ]
    }
   ],
   "source": [
    "experiments = []\n",
    "for experiment in db.runs.find(\n",
    "        {\n",
    "         \"experiment.name\": \"dcase2020_task2_classification\"\n",
    "        }\n",
    "):\n",
    "    if experiment['config'].get('descriptor') == 'per_mic_normalized': #'not_normalized':\n",
    "        experiments.append([\n",
    "            experiment['config']['machine_type'], \n",
    "            experiment['config']['machine_id'], \n",
    "            experiment['result']['auroc_max'] if experiment['result'] else 0,\n",
    "            experiment['result']['pauroc_max'] if experiment['result'] else 0,\n",
    "        ])\n",
    "print(\"{} experiments loaded\".format(len(experiments)))  \n",
    "\n",
    "# assert len(experiments) == 23\n",
    "\n",
    "\n",
    "print(\"| Machine | Type | ID   | AUC        | to BL      | AUC        | to BL      |\")\n",
    "print(\"| ------- | :--- | :--- | ---------- | ---------- | ---------- | ---------- |\")\n",
    "for e in experiments:\n",
    "        print(\"|{}\\t| {}\\t|{}\\t|{:0.4f}\\t|{:+0.4f}\\t|{:0.4f}\\t|{:+1.4f}\\t|\".format(\n",
    "            INVERSE_CLASS_MAP[e[0]],\n",
    "            e[0],\n",
    "            e[1],\n",
    "            e[2],\n",
    "            e[2] - baseline_auc[e[0]][e[1]],\n",
    "            e[3],\n",
    "            e[3] - baseline_pauc[e[0]][e[1]]\n",
    "        ))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "import dcase2020_task2\n",
    "\n",
    "from pymongo import MongoClient\n",
    "import numpy as np\n",
    "\n",
    "INVERSE_CLASS_MAP = {\n",
    "    0: 'fan',\n",
    "    1: 'pump',\n",
    "    2: 'slider',\n",
    "    3: 'ToyCar',\n",
    "    4: 'ToyConveyor',\n",
    "    5: 'valve'\n",
    "}\n",
    "\n",
    "baseline_auc = {\n",
    "    0: {0: 0.5441, 2: 0.7340, 4: 0.6161, 6: 0.7392},\n",
    "    1: {0: 0.6715, 2: 0.6153, 4: 0.8833, 6: 0.7455},\n",
    "    2: {0: 0.9619, 2: 0.7897, 4: 0.9430, 6: 0.6959},\n",
    "    3: {1: 0.8136, 2: 0.8597, 3: 0.6330, 4: 0.8445},\n",
    "    4: {1: 0.7807, 2: 0.6416, 3: 0.7535},\n",
    "    5: {0: 0.6876, 2: 0.6818, 4: 0.7430, 6: 0.5390}\n",
    "}\n",
    "\n",
    "baseline_pauc = {\n",
    "    0: {0: 0.4937, 2: 0.5481, 4: 0.5326, 6: 0.5235},\n",
    "    1: {0: 0.5674, 2: 0.5810, 4: 0.6710, 6: 0.5802},\n",
    "    2: {0: 0.8144, 2: 0.6368, 4: 0.7198, 6: 0.4902},\n",
    "    3: {1: 0.6840, 2: 0.7772, 3: 0.5521, 4: 0.6897},\n",
    "    4: {1: 0.6425, 2: 0.5601, 3: 0.6103},\n",
    "    5: {0: 0.5170, 2: 0.5183, 4: 0.5197, 6: 0.4843}\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "client = MongoClient('mongodb://student2.cp.jku.at:27017/')\n",
    "db = client.dcase2020_task2\n",
    "db.list_collection_names()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "experiments = []\n",
    "for experiment in db.runs.find(\n",
    "        {\n",
    "         \"experiment.name\": \"dcase2020_task2_classification\"\n",
    "        }\n",
    "):\n",
    "    if experiment['config'].get('descriptor') == 'not_normalized':\n",
    "        experiments.append([\n",
    "            experiment['config']['machine_type'], \n",
    "            experiment['config']['machine_id'], \n",
    "            experiment['result']['auroc_mean'] if experiment['result'] else 0,\n",
    "            experiment['result']['pauroc_mean'] if experiment['result'] else 0,\n",
    "        ])\n",
    "print(\"{} experiments loaded\".format(len(experiments)))  \n",
    "\n",
    "# assert len(experiments) == 23\n",
    "\n",
    "print(\"| Machine | Type | ID   | AUC        | to BL      | AUC        | to BL      |\")\n",
    "print(\"| ------- | :--- | :--- | ---------- | ---------- | ---------- | ---------- |\")\n",
    "for e in experiments:\n",
    "        print(\"|{}\\t| {}\\t|{}\\t|{:0.4f}\\t|{:+0.4f}\\t|{:0.4f}\\t|{:+1.4f}\\t|\".format(\n",
    "            INVERSE_CLASS_MAP[e[0]],\n",
    "            e[0],\n",
    "            e[1],\n",
    "            e[2],\n",
    "            e[2] - baseline_auc[e[0]][e[1]],\n",
    "            e[3],\n",
    "            e[3] - baseline_pauc[e[0]][e[1]]\n",
    "        ))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "pycharm": {
     "is_executing": false,
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "22 experiments loaded\n",
      "| Machine | Type | ID   | AUC        | to BL      | AUC        | to BL      |\n",
      "| ------- | :--- | :--- | ---------- | ---------- | ---------- | ---------- |\n",
      "|fan\t| 0\t|0\t|0.6683\t|+0.1242\t|0.6060\t|+0.1123\t|\n",
      "|fan\t| 0\t|2\t|0.9883\t|+0.2543\t|0.9456\t|+0.3975\t|\n",
      "|fan\t| 0\t|4\t|0.7859\t|+0.1698\t|0.6636\t|+0.1310\t|\n",
      "|fan\t| 0\t|6\t|0.7462\t|+0.0070\t|0.8455\t|+0.3220\t|\n",
      "|pump\t| 1\t|0\t|0.8785\t|+0.2070\t|0.7284\t|+0.1610\t|\n",
      "|pump\t| 1\t|2\t|0.5635\t|-0.0518\t|0.5462\t|-0.0348\t|\n",
      "|pump\t| 1\t|4\t|0.9995\t|+0.1162\t|0.9974\t|+0.3264\t|\n",
      "|pump\t| 1\t|6\t|0.9560\t|+0.2105\t|0.8653\t|+0.2851\t|\n",
      "|slider\t| 2\t|0\t|0.9679\t|+0.0060\t|0.8743\t|+0.0599\t|\n",
      "|slider\t| 2\t|2\t|0.8813\t|+0.0916\t|0.6339\t|-0.0029\t|\n",
      "|slider\t| 2\t|4\t|0.9905\t|+0.0475\t|0.9500\t|+0.2302\t|\n",
      "|slider\t| 2\t|6\t|0.8336\t|+0.1377\t|0.5577\t|+0.0675\t|\n",
      "|ToyCar\t| 3\t|1\t|0.6665\t|-0.1471\t|0.6647\t|-0.0193\t|\n",
      "|ToyCar\t| 3\t|2\t|0.9024\t|+0.0427\t|0.8195\t|+0.0423\t|\n",
      "|ToyCar\t| 3\t|3\t|0.9814\t|+0.3484\t|0.9271\t|+0.3750\t|\n",
      "|ToyCar\t| 3\t|4\t|0.9982\t|+0.1537\t|0.9904\t|+0.3007\t|\n",
      "|ToyConveyor\t| 4\t|1\t|0.8454\t|+0.0647\t|0.7466\t|+0.1041\t|\n",
      "|ToyConveyor\t| 4\t|2\t|0.6047\t|-0.0369\t|0.5448\t|-0.0153\t|\n",
      "|ToyConveyor\t| 4\t|3\t|0.7071\t|-0.0464\t|0.5888\t|-0.0215\t|\n",
      "|valve\t| 5\t|0\t|0.9876\t|+0.3000\t|0.9518\t|+0.4348\t|\n",
      "|valve\t| 5\t|2\t|0.7835\t|+0.1017\t|0.5197\t|+0.0014\t|\n",
      "|valve\t| 5\t|4\t|0.0000\t|-0.7430\t|0.0000\t|-0.5197\t|\n"
     ]
    }
   ],
   "source": [
    "experiments = []\n",
    "for experiment in db.runs.find(\n",
    "        {\n",
    "         \"experiment.name\": \"dcase2020_task2_classification\"\n",
    "        }\n",
    "):\n",
    "    if experiment['config'].get('descriptor') == 'per_mic_normalized': #'not_normalized':\n",
    "        experiments.append([\n",
    "            experiment['config']['machine_type'], \n",
    "            experiment['config']['machine_id'], \n",
    "            experiment['result']['auroc_mean'] if experiment['result'] else 0,\n",
    "            experiment['result']['pauroc_mean'] if experiment['result'] else 0,\n",
    "        ])\n",
    "print(\"{} experiments loaded\".format(len(experiments)))  \n",
    "\n",
    "# assert len(experiments) == 23\n",
    "\n",
    "\n",
    "print(\"| Machine | Type | ID   | AUC        | to BL      | AUC        | to BL      |\")\n",
    "print(\"| ------- | :--- | :--- | ---------- | ---------- | ---------- | ---------- |\")\n",
    "for e in experiments:\n",
    "        print(\"|{}\\t| {}\\t|{}\\t|{:0.4f}\\t|{:+0.4f}\\t|{:0.4f}\\t|{:+1.4f}\\t|\".format(\n",
    "            INVERSE_CLASS_MAP[e[0]],\n",
    "            e[0],\n",
    "            e[1],\n",
    "            e[2],\n",
    "            e[2] - baseline_auc[e[0]][e[1]],\n",
    "            e[3],\n",
    "            e[3] - baseline_pauc[e[0]][e[1]]\n",
    "        ))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "import dcase2020_task2\n",
    "\n",
    "from pymongo import MongoClient\n",
    "import numpy as np\n",
    "\n",
    "INVERSE_CLASS_MAP = {\n",
    "    0: 'fan',\n",
    "    1: 'pump',\n",
    "    2: 'slider',\n",
    "    3: 'ToyCar',\n",
    "    4: 'ToyConveyor',\n",
    "    5: 'valve'\n",
    "}\n",
    "\n",
    "baseline_auc = {\n",
    "    0: {0: 0.5441, 2: 0.7340, 4: 0.6161, 6: 0.7392},\n",
    "    1: {0: 0.6715, 2: 0.6153, 4: 0.8833, 6: 0.7455},\n",
    "    2: {0: 0.9619, 2: 0.7897, 4: 0.9430, 6: 0.6959},\n",
    "    3: {1: 0.8136, 2: 0.8597, 3: 0.6330, 4: 0.8445},\n",
    "    4: {1: 0.7807, 2: 0.6416, 3: 0.7535},\n",
    "    5: {0: 0.6876, 2: 0.6818, 4: 0.7430, 6: 0.5390}\n",
    "}\n",
    "\n",
    "baseline_pauc = {\n",
    "    0: {0: 0.4937, 2: 0.5481, 4: 0.5326, 6: 0.5235},\n",
    "    1: {0: 0.5674, 2: 0.5810, 4: 0.6710, 6: 0.5802},\n",
    "    2: {0: 0.8144, 2: 0.6368, 4: 0.7198, 6: 0.4902},\n",
    "    3: {1: 0.6840, 2: 0.7772, 3: 0.5521, 4: 0.6897},\n",
    "    4: {1: 0.6425, 2: 0.5601, 3: 0.6103},\n",
    "    5: {0: 0.5170, 2: 0.5183, 4: 0.5197, 6: 0.4843}\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "client = MongoClient('mongodb://student2.cp.jku.at:27017/')\n",
    "db = client.dcase2020_task2\n",
    "db.list_collection_names()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "pycharm": {
     "is_executing": false,
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "24 experiments loaded\n",
      "| Machine | Type | ID   | AUC        | to BL      | AUC        | to BL      |\n",
      "| ------- | :--- | :--- | ---------- | ---------- | ---------- | ---------- |\n",
      "|fan\t| 0\t|0\t|0.5857\t|+0.0416\t|0.5846\t|+0.0909\t|\n",
      "|fan\t| 0\t|2\t|0.9865\t|+0.2525\t|0.9315\t|+0.3834\t|\n",
      "|fan\t| 0\t|4\t|0.7753\t|+0.1592\t|0.6568\t|+0.1242\t|\n",
      "|fan\t| 0\t|6\t|0.7520\t|+0.0128\t|0.8455\t|+0.3220\t|\n",
      "|pump\t| 1\t|0\t|0.8373\t|+0.1658\t|0.7126\t|+0.1452\t|\n",
      "|pump\t| 1\t|2\t|0.5577\t|-0.0576\t|0.5225\t|-0.0585\t|\n",
      "|pump\t| 1\t|4\t|0.9995\t|+0.1162\t|0.9974\t|+0.3264\t|\n",
      "|pump\t| 1\t|6\t|0.9311\t|+0.1856\t|0.8354\t|+0.2552\t|\n",
      "|slider\t| 2\t|0\t|0.9946\t|+0.0327\t|0.9821\t|+0.1677\t|\n",
      "|slider\t| 2\t|2\t|0.7940\t|+0.0043\t|0.5494\t|-0.0874\t|\n",
      "|slider\t| 2\t|4\t|0.9953\t|+0.0523\t|0.9752\t|+0.2554\t|\n",
      "|slider\t| 2\t|6\t|0.8202\t|+0.1243\t|0.5795\t|+0.0893\t|\n",
      "|ToyCar\t| 3\t|1\t|0.6782\t|-0.1354\t|0.6485\t|-0.0355\t|\n",
      "|ToyCar\t| 3\t|2\t|0.8749\t|+0.0152\t|0.7847\t|+0.0075\t|\n",
      "|ToyCar\t| 3\t|3\t|0.9746\t|+0.3416\t|0.9001\t|+0.3480\t|\n",
      "|ToyCar\t| 3\t|4\t|0.0000\t|-0.8445\t|0.0000\t|-0.6897\t|\n",
      "|ToyCar\t| 3\t|4\t|0.9975\t|+0.1530\t|0.9867\t|+0.2970\t|\n",
      "|ToyConveyor\t| 4\t|1\t|0.8521\t|+0.0714\t|0.7509\t|+0.1084\t|\n",
      "|ToyConveyor\t| 4\t|2\t|0.5919\t|-0.0497\t|0.5407\t|-0.0194\t|\n",
      "|ToyConveyor\t| 4\t|3\t|0.7277\t|-0.0258\t|0.6033\t|-0.0070\t|\n",
      "|valve\t| 5\t|0\t|0.9838\t|+0.2962\t|0.9288\t|+0.4118\t|\n",
      "|valve\t| 5\t|2\t|0.7559\t|+0.0741\t|0.5048\t|-0.0135\t|\n",
      "|valve\t| 5\t|4\t|0.7259\t|-0.0171\t|0.5373\t|+0.0176\t|\n",
      "|valve\t| 5\t|6\t|0.7828\t|+0.2438\t|0.5623\t|+0.0780\t|\n"
     ]
    }
   ],
   "source": [
    "experiments = []\n",
    "for experiment in db.runs.find(\n",
    "        {\n",
    "         \"experiment.name\": \"dcase2020_task2_classification\"\n",
    "        }\n",
    "):\n",
    "    if experiment['config'].get('descriptor') == 'not_normalized':\n",
    "        experiments.append([\n",
    "            experiment['config']['machine_type'], \n",
    "            experiment['config']['machine_id'], \n",
    "            experiment['result']['auroc_mean'] if experiment['result'] else 0,\n",
    "            experiment['result']['pauroc_mean'] if experiment['result'] else 0,\n",
    "        ])\n",
    "print(\"{} experiments loaded\".format(len(experiments)))  \n",
    "\n",
    "# assert len(experiments) == 23\n",
    "\n",
    "print(\"| Machine | Type | ID   | AUC        | to BL      | AUC        | to BL      |\")\n",
    "print(\"| ------- | :--- | :--- | ---------- | ---------- | ---------- | ---------- |\")\n",
    "for e in experiments:\n",
    "        print(\"|{}\\t| {}\\t|{}\\t|{:0.4f}\\t|{:+0.4f}\\t|{:0.4f}\\t|{:+1.4f}\\t|\".format(\n",
    "            INVERSE_CLASS_MAP[e[0]],\n",
    "            e[0],\n",
    "            e[1],\n",
    "            e[2],\n",
    "            e[2] - baseline_auc[e[0]][e[1]],\n",
    "            e[3],\n",
    "            e[3] - baseline_pauc[e[0]][e[1]]\n",
    "        ))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "pycharm": {
     "is_executing": false,
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "23 experiments loaded\n",
      "| Machine | Type | ID   | AUC        | to BL      | AUC        | to BL      |\n",
      "| ------- | :--- | :--- | ---------- | ---------- | ---------- | ---------- |\n",
      "|fan\t| 0\t|0\t|0.6683\t|+0.1242\t|0.6060\t|+0.1123\t|\n",
      "|fan\t| 0\t|2\t|0.9883\t|+0.2543\t|0.9456\t|+0.3975\t|\n",
      "|fan\t| 0\t|4\t|0.7859\t|+0.1698\t|0.6636\t|+0.1310\t|\n",
      "|fan\t| 0\t|6\t|0.7462\t|+0.0070\t|0.8455\t|+0.3220\t|\n",
      "|pump\t| 1\t|0\t|0.8785\t|+0.2070\t|0.7284\t|+0.1610\t|\n",
      "|pump\t| 1\t|2\t|0.5635\t|-0.0518\t|0.5462\t|-0.0348\t|\n",
      "|pump\t| 1\t|4\t|0.9995\t|+0.1162\t|0.9974\t|+0.3264\t|\n",
      "|pump\t| 1\t|6\t|0.9560\t|+0.2105\t|0.8653\t|+0.2851\t|\n",
      "|slider\t| 2\t|0\t|0.9679\t|+0.0060\t|0.8743\t|+0.0599\t|\n",
      "|slider\t| 2\t|2\t|0.8813\t|+0.0916\t|0.6339\t|-0.0029\t|\n",
      "|slider\t| 2\t|4\t|0.9905\t|+0.0475\t|0.9500\t|+0.2302\t|\n",
      "|slider\t| 2\t|6\t|0.8336\t|+0.1377\t|0.5577\t|+0.0675\t|\n",
      "|ToyCar\t| 3\t|1\t|0.6665\t|-0.1471\t|0.6647\t|-0.0193\t|\n",
      "|ToyCar\t| 3\t|2\t|0.9024\t|+0.0427\t|0.8195\t|+0.0423\t|\n",
      "|ToyCar\t| 3\t|3\t|0.9814\t|+0.3484\t|0.9271\t|+0.3750\t|\n",
      "|ToyCar\t| 3\t|4\t|0.9982\t|+0.1537\t|0.9904\t|+0.3007\t|\n",
      "|ToyConveyor\t| 4\t|1\t|0.8454\t|+0.0647\t|0.7466\t|+0.1041\t|\n",
      "|ToyConveyor\t| 4\t|2\t|0.6047\t|-0.0369\t|0.5448\t|-0.0153\t|\n",
      "|ToyConveyor\t| 4\t|3\t|0.7071\t|-0.0464\t|0.5888\t|-0.0215\t|\n",
      "|valve\t| 5\t|0\t|0.9876\t|+0.3000\t|0.9518\t|+0.4348\t|\n",
      "|valve\t| 5\t|2\t|0.7835\t|+0.1017\t|0.5197\t|+0.0014\t|\n",
      "|valve\t| 5\t|4\t|0.7010\t|-0.0420\t|0.5289\t|+0.0092\t|\n",
      "|valve\t| 5\t|6\t|0.8042\t|+0.2652\t|0.5425\t|+0.0582\t|\n"
     ]
    }
   ],
   "source": [
    "experiments = []\n",
    "for experiment in db.runs.find(\n",
    "        {\n",
    "         \"experiment.name\": \"dcase2020_task2_classification\"\n",
    "        }\n",
    "):\n",
    "    if experiment['config'].get('descriptor') == 'per_mic_normalized': #'not_normalized':\n",
    "        experiments.append([\n",
    "            experiment['config']['machine_type'], \n",
    "            experiment['config']['machine_id'], \n",
    "            experiment['result']['auroc_mean'] if experiment['result'] else 0,\n",
    "            experiment['result']['pauroc_mean'] if experiment['result'] else 0,\n",
    "        ])\n",
    "print(\"{} experiments loaded\".format(len(experiments)))  \n",
    "\n",
    "# assert len(experiments) == 23\n",
    "\n",
    "\n",
    "print(\"| Machine | Type | ID   | AUC        | to BL      | AUC        | to BL      |\")\n",
    "print(\"| ------- | :--- | :--- | ---------- | ---------- | ---------- | ---------- |\")\n",
    "for e in experiments:\n",
    "        print(\"|{}\\t| {}\\t|{}\\t|{:0.4f}\\t|{:+0.4f}\\t|{:0.4f}\\t|{:+1.4f}\\t|\".format(\n",
    "            INVERSE_CLASS_MAP[e[0]],\n",
    "            e[0],\n",
    "            e[1],\n",
    "            e[2],\n",
    "            e[2] - baseline_auc[e[0]][e[1]],\n",
    "            e[3],\n",
    "            e[3] - baseline_pauc[e[0]][e[1]]\n",
    "        ))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "pycharm": {
     "is_executing": false,
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "23 experiments loaded\n",
      "| Machine | Type | ID   | AUC        | to BL      | AUC        | to BL      |\n",
      "| ------- | :--- | :--- | ---------- | ---------- | ---------- | ---------- |\n",
      "|fan\t| 0\t|0\t|0.6703\t|+0.1262\t|0.5874\t|+0.0937\t|\n",
      "|fan\t| 0\t|2\t|0.9831\t|+0.2491\t|0.9264\t|+0.3783\t|\n",
      "|fan\t| 0\t|4\t|0.7925\t|+0.1764\t|0.6645\t|+0.1319\t|\n",
      "|fan\t| 0\t|6\t|0.7467\t|+0.0075\t|0.8455\t|+0.3220\t|\n",
      "|pump\t| 1\t|0\t|0.8555\t|+0.1840\t|0.7022\t|+0.1348\t|\n",
      "|pump\t| 1\t|2\t|0.5559\t|-0.0594\t|0.5386\t|-0.0424\t|\n",
      "|pump\t| 1\t|4\t|0.9995\t|+0.1162\t|0.9974\t|+0.3264\t|\n",
      "|pump\t| 1\t|6\t|0.9425\t|+0.1970\t|0.8369\t|+0.2567\t|\n",
      "|slider\t| 2\t|0\t|0.9773\t|+0.0154\t|0.9117\t|+0.0973\t|\n",
      "|slider\t| 2\t|2\t|0.8856\t|+0.0959\t|0.6542\t|+0.0174\t|\n",
      "|slider\t| 2\t|4\t|0.9857\t|+0.0427\t|0.9249\t|+0.2051\t|\n",
      "|slider\t| 2\t|6\t|0.8374\t|+0.1415\t|0.5571\t|+0.0669\t|\n",
      "|ToyCar\t| 3\t|1\t|0.6835\t|-0.1301\t|0.6569\t|-0.0271\t|\n",
      "|ToyCar\t| 3\t|2\t|0.8717\t|+0.0120\t|0.7941\t|+0.0169\t|\n",
      "|ToyCar\t| 3\t|3\t|0.9855\t|+0.3525\t|0.9338\t|+0.3817\t|\n",
      "|ToyCar\t| 3\t|4\t|0.9983\t|+0.1538\t|0.9913\t|+0.3016\t|\n",
      "|ToyConveyor\t| 4\t|1\t|0.8592\t|+0.0785\t|0.7649\t|+0.1224\t|\n",
      "|ToyConveyor\t| 4\t|2\t|0.5898\t|-0.0518\t|0.5401\t|-0.0200\t|\n",
      "|ToyConveyor\t| 4\t|3\t|0.7115\t|-0.0420\t|0.5929\t|-0.0174\t|\n",
      "|valve\t| 5\t|0\t|0.9913\t|+0.3037\t|0.9655\t|+0.4485\t|\n",
      "|valve\t| 5\t|2\t|0.7976\t|+0.1158\t|0.5561\t|+0.0378\t|\n",
      "|valve\t| 5\t|4\t|0.7020\t|-0.0410\t|0.5373\t|+0.0176\t|\n",
      "|valve\t| 5\t|6\t|0.7919\t|+0.2529\t|0.5500\t|+0.0657\t|\n"
     ]
    }
   ],
   "source": [
    "experiments = []\n",
    "for experiment in db.runs.find(\n",
    "        {\n",
    "         \"experiment.name\": \"dcase2020_task2_classification\"\n",
    "        }\n",
    "):\n",
    "    if experiment['config'].get('descriptor') == 'per_mic_norm_same_mic_comp': #'not_normalized':\n",
    "        experiments.append([\n",
    "            experiment['config']['machine_type'], \n",
    "            experiment['config']['machine_id'], \n",
    "            experiment['result']['auroc_mean'] if experiment['result'] else 0,\n",
    "            experiment['result']['pauroc_mean'] if experiment['result'] else 0,\n",
    "        ])\n",
    "print(\"{} experiments loaded\".format(len(experiments)))  \n",
    "# assert len(experiments) == 23\n",
    "print(\"| Machine | Type | ID   | AUC        | to BL      | AUC        | to BL      |\")\n",
    "print(\"| ------- | :--- | :--- | ---------- | ---------- | ---------- | ---------- |\")\n",
    "for e in experiments:\n",
    "        print(\"|{}\\t| {}\\t|{}\\t|{:0.4f}\\t|{:+0.4f}\\t|{:0.4f}\\t|{:+1.4f}\\t|\".format(\n",
    "            INVERSE_CLASS_MAP[e[0]],\n",
    "            e[0],\n",
    "            e[1],\n",
    "            e[2],\n",
    "            e[2] - baseline_auc[e[0]][e[1]],\n",
    "            e[3],\n",
    "            e[3] - baseline_pauc[e[0]][e[1]]\n",
    "        ))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  },
  "pycharm": {
   "stem_cell": {
    "cell_type": "raw",
    "metadata": {
     "collapsed": false
    },
    "source": []
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}