Commit fe17fc86 authored by Shreyan Chowdhury's avatar Shreyan Chowdhury

commit for shake and fa

parent b9a557b2
......@@ -5,6 +5,7 @@ from pytorch_lightning.callbacks import EarlyStopping, ModelCheckpoint
from test_tube import Experiment, HyperOptArgumentParser
import os
from models.cp_resnet_locpad import Network
# from models.cp_preactbnresnetshakelocpad import Network
from utils import init_experiment, USE_GPU, dict_to_entry
......@@ -23,7 +24,7 @@ def main(hparams, gpu=None):
from utils import CURR_RUN_PATH # import these after init_experiment
logger = logging.getLogger(__name__)
logger = logging.getLogger('mw_log')
if not initialized:
logger.info(f"tensorboard --logdir={CURR_RUN_PATH}")
......@@ -36,14 +37,53 @@ def main(hparams, gpu=None):
for key in ['trials', 'hpc_exp_number', 'optimize_trials_parallel_gpu', 'generate_trials', 'optimize_parallel_cpu',
'optimize_parallel_gpu', 'optimize_parallel', 'load_from_trial_name']:
if key in hpdict: del hpdict[key]
logger.info(dict_to_entry(hpdict))
logger.info(dict_to_entry(hpdict, roundoff=None))
exp = Experiment(name=trial_name, save_dir=CURR_RUN_PATH)
model_config = {
batch_size = 10
model_config_basic = {
"input_shape": [batch_size, 1, -1, -1],
"arch": "",
"use_bn": True,
"multi_label": False,
"n_classes": 7,
"prediction_threshold": 0,
"depth": 26,
"base_channels": 64,
"stage1": {"maxpool": [1, 2, 4], "k1s": [3, 3, 3, 3, 3, 3], "k2s": [1, 3, 3, 3, 3, 3]},
"stage2": {"maxpool": [], "k1s": [3, 3, 3, 3, 3, 3], "k2s": [3, 3, 3, 3, 3, 3]},
"stage3": {"maxpool": [], "k1s": [3, 3, 3, 3, 3, 3], "k2s": [3, 3, 3, 3, 3, 3]},
"block_type": "basic",
"binary_classifier": True,
'validation_metrics': ['corr_avg'],
'test_metrics': ['corr_avg', 'corr']
}
model_config_shakefa = {
"arch":"cp_preactbnresnetshakelocpad",
"base_channels":128,
"binary_classifier":True,
"block_type":"basic",
"depth":26,
"input_shape":[10,1,-1,-1],
"multi_label":False,
"n_classes":7,
"prediction_threshold":0,
"stage1":{"k1s":[3,3,3,3], "k2s":[1,3,3,3], "maxpool":[1,2,4]},
"stage2":{"k1s":[1,1,1,1], "k2s":[1,1,1,1], "maxpool":[]},
"stage3":{"k1s":[1,1,1,1],"k2s":[1,1,1,1],"maxpool":[]},
"use_bn":True,
"weight_init":"fixup",
'validation_metrics': ['corr_avg'],
'test_metrics': ['corr_avg', 'corr']
}
model_config = model_config_basic
logger.info(dict_to_entry(model_config, roundoff=None))
# build model
model = Network(config=model_config, hparams=hparams, num_targets=7)
......@@ -126,5 +166,6 @@ if __name__=='__main__':
# hyperparams.optimize_parallel_gpu(main, gpu_ids=['0','1','2','3'])
for hparams in hyperparams.trials(18):
main(hparams)
main(hyperparams)
# for hparams in hyperparams.trials(18):
# main(hparams)
This diff is collapsed.
# coding: utf-8
import math
from models.shared_stuff import BasePtlModel
import torch.nn.functional as F
from datasets.midlevel import df_get_midlevel_set
......@@ -232,19 +233,27 @@ class BottleneckBlock(nn.Module):
return y
class Network(nn.Module):
class Network(BasePtlModel):
def __init__(self, config, hparams, num_targets=7):
super(Network, self).__init__()
super(Network, self).__init__(config, hparams)
self.logger = logging.getLogger('mw_log')
audio_path, csvs_path = path_midlevel_audio_dir, path_midlevel_annotations_dir
cache_x_name = '_ap_midlevel44k'
from torch.utils.data import random_split
dataset, dataset_length = df_get_midlevel_set('midlevel', os.path.join(csvs_path, 'annotations.csv'),
audio_path, cache_x_name)
self.trainset, self.validationset, self.testset = random_split(dataset, [int(i * dataset_length) for i in
[0.7, 0.2, 0.1]])
tr_dataset, tr_dataset_length = df_get_midlevel_set('midlevel', os.path.join(csvs_path, 'annotations.csv'),
os.path.join(csvs_path, 'metadata.csv'),
audio_path, cache_x_name, aljanaki=True, dset='train')
tst_dataset, tst_dataset_length = df_get_midlevel_set('midlevel', os.path.join(csvs_path, 'annotations.csv'),
os.path.join(csvs_path, 'metadata.csv'),
audio_path, cache_x_name, aljanaki=True, dset='test')
self.testset = tst_dataset
self.trainset, self.validationset = random_split(tr_dataset, [int(i * tr_dataset_length) for i in [0.98, 0.02]])
self.num_targets = num_targets
self.dataset_name = config.get("dataset_name")
self.loss_fn = config.get("loss_fn")
......@@ -406,6 +415,55 @@ class Network(nn.Module):
first_RUN = False
return logit
@classmethod
def load_from_metrics(cls, weights_path, config, tags_csv=None, on_gpu=True):
def load_hparams_from_tags_csv(tags_csv):
from argparse import Namespace
import pandas as pd
tags_df = pd.read_csv(tags_csv)
dic = tags_df.to_dict(orient='records')
ns_dict = {row['key']: convert(row['value']) for row in dic}
ns = Namespace(**ns_dict)
return ns
def convert(val):
constructors = [int, float, str]
if type(val) is str:
if val.lower() == 'true':
return True
if val.lower() == 'false':
return False
for c in constructors:
try:
return c(val)
except ValueError:
pass
return val
hparams = load_hparams_from_tags_csv(tags_csv)
hparams.__setattr__('on_gpu', on_gpu)
# load on CPU only to avoid OOM issues
# then its up to user to put back on GPUs
checkpoint = torch.load(weights_path, map_location=lambda storage, loc: storage)
# load the state_dict on the model automatically
model = cls(config, hparams, num_targets=7)
model.load_state_dict(checkpoint['state_dict'])
# give model a chance to load something
model.on_load_checkpoint(checkpoint)
return model
def log(self, msg):
self.logger.info(msg)
def my_loss(self, y_hat, y):
return F.mse_loss(y_hat, y)
......@@ -438,9 +496,8 @@ class Network(nn.Module):
metrics = self._compute_metrics(y, y_hat, self.validation_metrics)
metrics['val_loss'] = avg_loss
return {'val_loss': metrics['val_loss'],
'corr': metrics['corr_avg']}
self.log('Val: '+dict_to_entry(metrics, filter=['corr_avg']))
return metrics
def test_step(self, data_batch, batch_nb):
x, _, y = data_batch
......@@ -459,25 +516,29 @@ class Network(nn.Module):
for output in outputs:
y.append(output['y'])
y_hat.append(output['y_hat'])
y = np.concatenate(y)
y_hat = np.concatenate(y_hat)
test_metrics = self._compute_metrics(y, y_hat, self.test_metrics)
test_metrics['avg_test_loss'] = avg_test_loss
self.experiment.log(test_metrics)
# print(test_metrics)
# self.experiment.log(test_metrics)
self.log('Test: '+dict_to_entry(test_metrics, filter=['corr_avg', 'corr']))
return test_metrics
def configure_optimizers(self):
return [torch.optim.Adam(self.parameters(), lr=1e-4)] # from their code
return [torch.optim.Adam(self.parameters(), lr=self.hparams.learning_rate)] # from their code
@pl.data_loader
def train_dataloader(self):
return DataLoader(dataset=self.trainset, batch_size=32, shuffle=True)
return DataLoader(dataset=self.trainset, batch_size=self.hparams.batch_size, shuffle=True)
@pl.data_loader
def val_dataloader(self):
return DataLoader(dataset=self.validationset, batch_size=32, shuffle=True)
return DataLoader(dataset=self.validationset, batch_size=self.hparams.batch_size, shuffle=True)
@pl.data_loader
def test_dataloader(self):
return DataLoader(dataset=self.testset, batch_size=32, shuffle=True)
return DataLoader(dataset=self.testset, batch_size=self.hparams.batch_size, shuffle=True)
@staticmethod
def add_model_specific_args(parent_parser):
......@@ -486,10 +547,13 @@ class Network(nn.Module):
parser = HyperOptArgumentParser(strategy=parent_parser.strategy, parents=[parent_parser])
# network params
parser.opt_list('--dropout', default=0.8, type=float,
options=[0.2, 0.5, 0.8],
tunable=False)
parser.opt_list('--learning_rate', default=0.0001, type=float,
options=[0.00001, 0.0005, 0.001],
tunable=True)
parser.opt_list('--input_size', default=1024, options=[512, 1024], type=int, tunable=True)
parser.opt_list('--batch_size', default=8, options=[8, 16], type=int, tunable=True)
options=[0.00001, 0.0005, 0.001],
tunable=False)
# parser.opt_list('--input_size', default=1024, options=[512, 1024], type=int, tunable=False)
parser.opt_list('--batch_size', default=8, options=[8, 16], type=int, tunable=False)
return parser
return parser
\ No newline at end of file
......@@ -549,13 +549,13 @@ class ModelMidlevelBasic(BasePtlModel):
parser = HyperOptArgumentParser(strategy=parent_parser.strategy, parents=[parent_parser])
# network params
parser.opt_list('--dropout', default=0.3, type=float,
parser.opt_list('--dropout', default=0.8, type=float,
options=[0.2, 0.5, 0.8],
tunable=True)
tunable=False)
parser.opt_list('--learning_rate', default=0.0001, type=float,
options=[0.00001, 0.0005, 0.001],
tunable=True)
tunable=False)
# parser.opt_list('--input_size', default=1024, options=[512, 1024], type=int, tunable=False)
parser.opt_list('--batch_size', default=8, options=[8, 16], type=int, tunable=True)
parser.opt_list('--batch_size', default=8, options=[8, 16], type=int, tunable=False)
return parser
\ No newline at end of file
# -*- coding: utf-8 -*-
import torch
import torch.nn as nn
import torch.nn.functional as F
from torch.autograd import Variable
class ShakeShake(torch.autograd.Function):
@staticmethod
def forward(ctx, x1, x2, training=True):
if training:
alpha = torch.cuda.FloatTensor(x1.size(0)).uniform_()
alpha = alpha.view(alpha.size(0), 1, 1, 1).expand_as(x1)
else:
alpha = 0.5
return alpha * x1 + (1 - alpha) * x2
@staticmethod
def backward(ctx, grad_output):
beta = torch.cuda.FloatTensor(grad_output.size(0)).uniform_()
beta = beta.view(beta.size(0), 1, 1, 1).expand_as(grad_output)
beta = Variable(beta)
return beta * grad_output, (1 - beta) * grad_output, None
class Shortcut(nn.Module):
def __init__(self, in_ch, out_ch, stride):
super(Shortcut, self).__init__()
self.stride = stride
self.conv1 = nn.Conv2d(in_ch, out_ch // 2, 1, stride=1, padding=0, bias=False)
self.conv2 = nn.Conv2d(in_ch, out_ch // 2, 1, stride=1, padding=0, bias=False)
self.bn = nn.BatchNorm2d(out_ch)
def forward(self, x):
h = F.relu(x)
h1 = F.avg_pool2d(h, 1, self.stride)
h1 = self.conv1(h1)
h2 = F.avg_pool2d(F.pad(h, (-1, 1, -1, 1)), 1, self.stride)
h2 = self.conv2(h2)
h = torch.cat((h1, h2), 1)
return self.bn(h)
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment