Commit cc1e6825 authored by Paul Primus's avatar Paul Primus
Browse files

add micro seconds to experiment name

parent 0d234db8
......@@ -15,7 +15,7 @@
# Baseline Results
## Raw Normalized, Spec Normalized
## Spec Normalized, Raw Normalized
| Machine | Type | ID | AUC | to BL | AUC | to BL |
| ------- | :--- | :--- | ---------- | ---------- | ---------- | ---------- |
......@@ -43,7 +43,7 @@
|valve | 5 |4 |0.7339 |-0.0091 |0.5263 |+0.0066 |
|valve | 5 |6 |0.5888 |+0.0498 |0.4947 |+0.0104 |
## Raw Unnormalized, Spec Unnormalized
## Spec Unnormalized, Raw Unnormalized
| Machine | Type | ID | AUC | to BL | AUC | to BL |
| ------- | :--- | :--- | ---------- | ---------- | ---------- | ---------- |
......@@ -72,7 +72,7 @@
|valve | 5 |6 |0.4584 |-0.0806 |0.4961 |+0.0118 |
## Raw Normalized, Spec Unnormalized
## Spec Unnormalized, Raw Normalized
| Machine | Type | ID | AUC | to BL | AUC | to BL |
......@@ -100,3 +100,33 @@
|valve | 5 |2 |0.7115 |+0.0297 |0.5246 |+0.0063 |
|valve | 5 |4 |0.7423 |-0.0007 |0.5110 |-0.0087 |
|valve | 5 |6 |0.6381 |+0.0991 |0.5000 |+0.0157 |
# Classification
| Machine | Type | ID | AUC | to BL | AUC | to BL |
| ------- | :--- | :--- | ---------- | ---------- | ---------- | ---------- |
|fan | 0 |0 |0.6861 |+0.1420 |0.6139 |+0.1202 |
|fan | 0 |2 |0.9867 |+0.2527 |0.9370 |+0.3889 |
|fan | 0 |4 |0.8060 |+0.1899 |0.6676 |+0.1350 |
|fan | 0 |6 |0.7448 |+0.0056 |0.8455 |+0.3220 |
|pump | 1 |0 |0.8480 |+0.1765 |0.7030 |+0.1356 |
|pump | 1 |2 |0.5552 |-0.0601 |0.5586 |-0.0224 |
|pump | 1 |4 |0.9995 |+0.1162 |0.9974 |+0.3264 |
|pump | 1 |6 |0.9117 |+0.1662 |0.7632 |+0.1830 |
|slider | 2 |0 |0.5270 |-0.4349 |0.6415 |-0.1729 |
|slider | 2 |2 |0.7433 |-0.0464 |0.5346 |-0.1022 |
|slider | 2 |4 |0.9879 |+0.0449 |0.9364 |+0.2166 |
|slider | 2 |6 |0.8589 |+0.1630 |0.5707 |+0.0805 |
|ToyCar | 3 |1 |0.4801 |-0.3335 |0.4940 |-0.1900 |
|ToyCar | 3 |2 |0.4581 |-0.4016 |0.4854 |-0.2918 |
|ToyCar | 3 |3 |0.4100 |-0.2230 |0.4870 |-0.0651 |
|ToyCar | 3 |4 |0.3659 |-0.4786 |0.4772 |-0.2125 |
|ToyConveyor | 4 |1 |0.7495 |-0.0312 |0.6487 |+0.0062 |
|ToyConveyor | 4 |2 |0.5607 |-0.0809 |0.5252 |-0.0349 |
|ToyConveyor | 4 |3 |0.6587 |-0.0948 |0.5552 |-0.0551 |
|valve | 5 |0 |0.9791 |+0.2915 |0.9390 |+0.4220 |
|valve | 5 |2 |0.3698 |-0.3120 |0.4895 |-0.0288 |
|valve | 5 |4 |0.6561 |-0.0869 |0.5175 |-0.0022 |
|valve | 5 |6 |0.7242 |+0.1852 |0.5417 |+0.0574 |
......@@ -63,57 +63,11 @@
},
{
"cell_type": "code",
"execution_count": 111,
"execution_count": 116,
"outputs": [
{
"name": "stdout",
"text": [
"True\n",
"False\n",
"True\n",
"False\n",
"True\n",
"False\n",
"True\n",
"False\n",
"True\n",
"False\n",
"True\n",
"False\n",
"True\n",
"False\n",
"True\n",
"False\n",
"True\n",
"False\n",
"True\n",
"False\n",
"True\n",
"False\n",
"True\n",
"False\n",
"True\n",
"False\n",
"True\n",
"False\n",
"True\n",
"False\n",
"True\n",
"False\n",
"True\n",
"False\n",
"True\n",
"False\n",
"True\n",
"False\n",
"True\n",
"False\n",
"True\n",
"False\n",
"True\n",
"False\n",
"True\n",
"False\n",
"23 experiments loaded\n"
],
"output_type": "stream"
......@@ -123,19 +77,15 @@
"experiments = []\n",
"for experiment in db.runs.find(\n",
" {\n",
" \"experiment.name\": \"dcase2020_task2_baseline\"\n",
" \"experiment.name\": \"dcase2020_task2_classification\"\n",
" }\n",
" \n",
"):\n",
" if (experiment['config'].get('normalize', True) is True) and (experiment['config'].get('normalize_raw', True) is False):\n",
" print(experiment['config'].get('normalize', True))\n",
" print(experiment['config'].get('normalize_raw', True))\n",
" if (experiment['config'].get('normalize', True) is True) and (experiment['config'].get('normalize_raw', True) is True):\n",
" experiments.append([\n",
" experiment['config']['machine_type'], \n",
" experiment['config']['machine_id'], \n",
" experiment['result']['auroc_mean'],\n",
" experiment['result']['pauroc_mean'],\n",
" \n",
" ])\n",
"print(\"{} experiments loaded\".format(len(experiments))) \n",
"\n",
......@@ -151,36 +101,36 @@
},
{
"cell_type": "code",
"execution_count": 89,
"execution_count": 117,
"outputs": [
{
"name": "stdout",
"text": [
"| Machine | Type | ID | AUC | to BL | AUC | to BL |\n",
"| ------- | :--- | :--- | ---------- | ---------- | ---------- | ---------- |\n",
"|fan\t| 0\t|0\t|0.5516\t|+0.0075\t|0.5096\t|+0.0159\t|\n",
"|fan\t| 0\t|2\t|0.7285\t|-0.0055\t|0.5584\t|+0.0103\t|\n",
"|fan\t| 0\t|4\t|0.5530\t|-0.0631\t|0.5032\t|-0.0294\t|\n",
"|fan\t| 0\t|6\t|0.7911\t|+0.0519\t|0.6481\t|+0.1246\t|\n",
"|pump\t| 1\t|0\t|0.7281\t|+0.0566\t|0.5970\t|+0.0296\t|\n",
"|pump\t| 1\t|2\t|0.5812\t|-0.0341\t|0.5495\t|-0.0315\t|\n",
"|pump\t| 1\t|4\t|0.9266\t|+0.0433\t|0.7463\t|+0.0753\t|\n",
"|pump\t| 1\t|6\t|0.6832\t|-0.0623\t|0.6197\t|+0.0395\t|\n",
"|slider\t| 2\t|0\t|0.9423\t|-0.0196\t|0.7306\t|-0.0838\t|\n",
"|slider\t| 2\t|2\t|0.7222\t|-0.0675\t|0.5456\t|-0.0912\t|\n",
"|slider\t| 2\t|4\t|0.6449\t|-0.2981\t|0.5506\t|-0.1692\t|\n",
"|slider\t| 2\t|6\t|0.5434\t|-0.1525\t|0.5098\t|+0.0196\t|\n",
"|ToyCar\t| 3\t|1\t|0.7650\t|-0.0486\t|0.6409\t|-0.0431\t|\n",
"|ToyCar\t| 3\t|2\t|0.8162\t|-0.0435\t|0.6620\t|-0.1152\t|\n",
"|ToyCar\t| 3\t|3\t|0.6456\t|+0.0126\t|0.5514\t|-0.0007\t|\n",
"|ToyCar\t| 3\t|4\t|0.8892\t|+0.0447\t|0.7331\t|+0.0434\t|\n",
"|ToyConveyor\t| 4\t|1\t|0.6908\t|-0.0899\t|0.5599\t|-0.0826\t|\n",
"|ToyConveyor\t| 4\t|2\t|0.5987\t|-0.0429\t|0.5178\t|-0.0423\t|\n",
"|ToyConveyor\t| 4\t|3\t|0.6794\t|-0.0741\t|0.5444\t|-0.0659\t|\n",
"|valve\t| 5\t|0\t|0.5355\t|-0.1521\t|0.5033\t|-0.0137\t|\n",
"|valve\t| 5\t|2\t|0.5654\t|-0.1164\t|0.4855\t|-0.0328\t|\n",
"|valve\t| 5\t|4\t|0.4882\t|-0.2548\t|0.5162\t|-0.0035\t|\n",
"|valve\t| 5\t|6\t|0.4584\t|-0.0806\t|0.4961\t|+0.0118\t|\n"
"|fan\t| 0\t|0\t|0.6861\t|+0.1420\t|0.6139\t|+0.1202\t|\n",
"|fan\t| 0\t|2\t|0.9867\t|+0.2527\t|0.9370\t|+0.3889\t|\n",
"|fan\t| 0\t|4\t|0.8060\t|+0.1899\t|0.6676\t|+0.1350\t|\n",
"|fan\t| 0\t|6\t|0.7448\t|+0.0056\t|0.8455\t|+0.3220\t|\n",
"|pump\t| 1\t|0\t|0.8480\t|+0.1765\t|0.7030\t|+0.1356\t|\n",
"|pump\t| 1\t|2\t|0.5552\t|-0.0601\t|0.5586\t|-0.0224\t|\n",
"|pump\t| 1\t|4\t|0.9995\t|+0.1162\t|0.9974\t|+0.3264\t|\n",
"|pump\t| 1\t|6\t|0.9117\t|+0.1662\t|0.7632\t|+0.1830\t|\n",
"|slider\t| 2\t|0\t|0.5270\t|-0.4349\t|0.6415\t|-0.1729\t|\n",
"|slider\t| 2\t|2\t|0.7433\t|-0.0464\t|0.5346\t|-0.1022\t|\n",
"|slider\t| 2\t|4\t|0.9879\t|+0.0449\t|0.9364\t|+0.2166\t|\n",
"|slider\t| 2\t|6\t|0.8589\t|+0.1630\t|0.5707\t|+0.0805\t|\n",
"|ToyCar\t| 3\t|1\t|0.4801\t|-0.3335\t|0.4940\t|-0.1900\t|\n",
"|ToyCar\t| 3\t|2\t|0.4581\t|-0.4016\t|0.4854\t|-0.2918\t|\n",
"|ToyCar\t| 3\t|3\t|0.4100\t|-0.2230\t|0.4870\t|-0.0651\t|\n",
"|ToyCar\t| 3\t|4\t|0.3659\t|-0.4786\t|0.4772\t|-0.2125\t|\n",
"|ToyConveyor\t| 4\t|1\t|0.7495\t|-0.0312\t|0.6487\t|+0.0062\t|\n",
"|ToyConveyor\t| 4\t|2\t|0.5607\t|-0.0809\t|0.5252\t|-0.0349\t|\n",
"|ToyConveyor\t| 4\t|3\t|0.6587\t|-0.0948\t|0.5552\t|-0.0551\t|\n",
"|valve\t| 5\t|0\t|0.9791\t|+0.2915\t|0.9390\t|+0.4220\t|\n",
"|valve\t| 5\t|2\t|0.3698\t|-0.3120\t|0.4895\t|-0.0288\t|\n",
"|valve\t| 5\t|4\t|0.6561\t|-0.0869\t|0.5175\t|-0.0022\t|\n",
"|valve\t| 5\t|6\t|0.7242\t|+0.1852\t|0.5417\t|+0.0574\t|\n"
],
"output_type": "stream"
}
......
%% Cell type:code id: tags:
``` python
import dcase2020_task2
from pymongo import MongoClient
import numpy as np
INVERSE_CLASS_MAP = {
0: 'fan',
1: 'pump',
2: 'slider',
3: 'ToyCar',
4: 'ToyConveyor',
5: 'valve'
}
baseline_auc = {
0: {0: 0.5441, 2: 0.7340, 4: 0.6161, 6: 0.7392},
1: {0: 0.6715, 2: 0.6153, 4: 0.8833, 6: 0.7455},
2: {0: 0.9619, 2: 0.7897, 4: 0.9430, 6: 0.6959},
3: {1: 0.8136, 2: 0.8597, 3: 0.6330, 4: 0.8445},
4: {1: 0.7807, 2: 0.6416, 3: 0.7535},
5: {0: 0.6876, 2: 0.6818, 4: 0.7430, 6: 0.5390}
}
baseline_pauc = {
0: {0: 0.4937, 2: 0.5481, 4: 0.5326, 6: 0.5235},
1: {0: 0.5674, 2: 0.5810, 4: 0.6710, 6: 0.5802},
2: {0: 0.8144, 2: 0.6368, 4: 0.7198, 6: 0.4902},
3: {1: 0.6840, 2: 0.7772, 3: 0.5521, 4: 0.6897},
4: {1: 0.6425, 2: 0.5601, 3: 0.6103},
5: {0: 0.5170, 2: 0.5183, 4: 0.5197, 6: 0.4843}
}
```
%% Cell type:code id: tags:
``` python
client = MongoClient('mongodb://student2.cp.jku.at:27017/')
db = client.dcase2020_task2
db.list_collection_names()
```
%% Cell type:code id: tags:
``` python
experiments = []
for experiment in db.runs.find(
{
"experiment.name": "dcase2020_task2_baseline"
"experiment.name": "dcase2020_task2_classification"
}
):
if (experiment['config'].get('normalize', True) is True) and (experiment['config'].get('normalize_raw', True) is False):
print(experiment['config'].get('normalize', True))
print(experiment['config'].get('normalize_raw', True))
if (experiment['config'].get('normalize', True) is True) and (experiment['config'].get('normalize_raw', True) is True):
experiments.append([
experiment['config']['machine_type'],
experiment['config']['machine_id'],
experiment['result']['auroc_mean'],
experiment['result']['pauroc_mean'],
])
print("{} experiments loaded".format(len(experiments)))
assert len(experiments) == 23
```
%%%% Output: stream
True
False
True
False
True
False
True
False
True
False
True
False
True
False
True
False
True
False
True
False
True
False
True
False
True
False
True
False
True
False
True
False
True
False
True
False
True
False
True
False
True
False
True
False
True
False
23 experiments loaded
%% Cell type:code id: tags:
``` python
print("| Machine | Type | ID | AUC | to BL | AUC | to BL |")
print("| ------- | :--- | :--- | ---------- | ---------- | ---------- | ---------- |")
for e in experiments:
print("|{}\t| {}\t|{}\t|{:0.4f}\t|{:+0.4f}\t|{:0.4f}\t|{:+1.4f}\t|".format(
INVERSE_CLASS_MAP[e[0]],
e[0],
e[1],
e[2],
e[2] - baseline_auc[e[0]][e[1]],
e[3],
e[3] - baseline_pauc[e[0]][e[1]]
))
```
%%%% Output: stream
| Machine | Type | ID | AUC | to BL | AUC | to BL |
| ------- | :--- | :--- | ---------- | ---------- | ---------- | ---------- |
|fan | 0 |0 |0.5516 |+0.0075 |0.5096 |+0.0159 |
|fan | 0 |2 |0.7285 |-0.0055 |0.5584 |+0.0103 |
|fan | 0 |4 |0.5530 |-0.0631 |0.5032 |-0.0294 |
|fan | 0 |6 |0.7911 |+0.0519 |0.6481 |+0.1246 |
|pump | 1 |0 |0.7281 |+0.0566 |0.5970 |+0.0296 |
|pump | 1 |2 |0.5812 |-0.0341 |0.5495 |-0.0315 |
|pump | 1 |4 |0.9266 |+0.0433 |0.7463 |+0.0753 |
|pump | 1 |6 |0.6832 |-0.0623 |0.6197 |+0.0395 |
|slider | 2 |0 |0.9423 |-0.0196 |0.7306 |-0.0838 |
|slider | 2 |2 |0.7222 |-0.0675 |0.5456 |-0.0912 |
|slider | 2 |4 |0.6449 |-0.2981 |0.5506 |-0.1692 |
|slider | 2 |6 |0.5434 |-0.1525 |0.5098 |+0.0196 |
|ToyCar | 3 |1 |0.7650 |-0.0486 |0.6409 |-0.0431 |
|ToyCar | 3 |2 |0.8162 |-0.0435 |0.6620 |-0.1152 |
|ToyCar | 3 |3 |0.6456 |+0.0126 |0.5514 |-0.0007 |
|ToyCar | 3 |4 |0.8892 |+0.0447 |0.7331 |+0.0434 |
|ToyConveyor | 4 |1 |0.6908 |-0.0899 |0.5599 |-0.0826 |
|ToyConveyor | 4 |2 |0.5987 |-0.0429 |0.5178 |-0.0423 |
|ToyConveyor | 4 |3 |0.6794 |-0.0741 |0.5444 |-0.0659 |
|valve | 5 |0 |0.5355 |-0.1521 |0.5033 |-0.0137 |
|valve | 5 |2 |0.5654 |-0.1164 |0.4855 |-0.0328 |
|valve | 5 |4 |0.4882 |-0.2548 |0.5162 |-0.0035 |
|valve | 5 |6 |0.4584 |-0.0806 |0.4961 |+0.0118 |
|fan | 0 |0 |0.6861 |+0.1420 |0.6139 |+0.1202 |
|fan | 0 |2 |0.9867 |+0.2527 |0.9370 |+0.3889 |
|fan | 0 |4 |0.8060 |+0.1899 |0.6676 |+0.1350 |
|fan | 0 |6 |0.7448 |+0.0056 |0.8455 |+0.3220 |
|pump | 1 |0 |0.8480 |+0.1765 |0.7030 |+0.1356 |
|pump | 1 |2 |0.5552 |-0.0601 |0.5586 |-0.0224 |
|pump | 1 |4 |0.9995 |+0.1162 |0.9974 |+0.3264 |
|pump | 1 |6 |0.9117 |+0.1662 |0.7632 |+0.1830 |
|slider | 2 |0 |0.5270 |-0.4349 |0.6415 |-0.1729 |
|slider | 2 |2 |0.7433 |-0.0464 |0.5346 |-0.1022 |
|slider | 2 |4 |0.9879 |+0.0449 |0.9364 |+0.2166 |
|slider | 2 |6 |0.8589 |+0.1630 |0.5707 |+0.0805 |
|ToyCar | 3 |1 |0.4801 |-0.3335 |0.4940 |-0.1900 |
|ToyCar | 3 |2 |0.4581 |-0.4016 |0.4854 |-0.2918 |
|ToyCar | 3 |3 |0.4100 |-0.2230 |0.4870 |-0.0651 |
|ToyCar | 3 |4 |0.3659 |-0.4786 |0.4772 |-0.2125 |
|ToyConveyor | 4 |1 |0.7495 |-0.0312 |0.6487 |+0.0062 |
|ToyConveyor | 4 |2 |0.5607 |-0.0809 |0.5252 |-0.0349 |
|ToyConveyor | 4 |3 |0.6587 |-0.0948 |0.5552 |-0.0551 |
|valve | 5 |0 |0.9791 |+0.2915 |0.9390 |+0.4220 |
|valve | 5 |2 |0.3698 |-0.3120 |0.4895 |-0.0288 |
|valve | 5 |4 |0.6561 |-0.0869 |0.5175 |-0.0022 |
|valve | 5 |6 |0.7242 |+0.1852 |0.5417 |+0.0574 |
%% Cell type:code id: tags:
``` python
```
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment