Commit ba5869eb authored by Shreyan Chowdhury's avatar Shreyan Chowdhury

run status, full song

parent b3a8c6f7
from utils import USE_GPU, init_experiment
from utils import USE_GPU, init_experiment, exit_experiment
from pytorch_lightning import Trainer
from pytorch_lightning.callbacks import EarlyStopping, ModelCheckpoint
from test_tube import Experiment, HyperOptArgumentParser
......@@ -60,13 +60,29 @@ def run(hparams):
nb_sanity_val_steps=0) # don't run sanity validation run
else:
trainer = Trainer(experiment=exp, max_nb_epochs=1, train_percent_check=0.1,
fast_dev_run=True)
fast_dev_run=True, nb_sanity_val_steps=0)
model = Network(num_class=56, config=model_config, hparams=hparams)
print(model)
trainer.fit(model)
trainer.test()
try:
trainer.fit(model)
except KeyboardInterrupt:
logger.info("Training interrupted")
except:
logger.exception(msg="Error occurred during train!")
exit_experiment('failed', exp)
try:
logger.info("Starting test...")
trainer.test()
except KeyboardInterrupt:
logger.info("Exiting...")
exit_experiment('stopped', exp)
except:
logger.exception(msg="Error occurred during test!")
exit_experiment('failed', exp)
if __name__ == '__main__':
......
......@@ -71,16 +71,26 @@ class CRNN(BasePtlModel):
# classifier
x = x.view(-1, x.size(0), 32)
<<<<<<< Updated upstream
=======
# print(x.shape)
>>>>>>> Stashed changes
return x
def rnn_forward(x):
<<<<<<< Updated upstream
=======
# print(x.squeeze().shape)
x = x.squeeze()
>>>>>>> Stashed changes
x = self.gru1(x)[1][1] # TODO: Check if this is correct
x = self.dropout(x)
logit = nn.Sigmoid()(self.dense(x))
return logit
def extract_features(song_idx, song_length):
<<<<<<< Updated upstream
song_feats = []
for l in range(song_length//self.input_size + 1):
data = h5data[song_idx + l*self.input_size:song_idx + min(song_length, (l + 1) * self.input_size)].transpose()
......@@ -90,6 +100,23 @@ class CRNN(BasePtlModel):
except:
song_feats.append(cnn_forward(torch.tensor([[data]], device=torch.device('cpu'))))
=======
# print(song_idx, song_length)
song_length = 2560
song_feats = []
for l in range(song_length//self.input_size + 1):
data = h5data[song_idx + l*self.input_size:song_idx + min(song_length, (l + 1) * self.input_size)].transpose()
if data.shape[1] < self.input_size*0.25:
continue
data = np.pad(data, ((0, 0), (0, self.input_size-data.shape[1])), mode='wrap')
try:
song_feats.append(cnn_forward(torch.tensor([[data]], device=torch.device('cuda'))))
except AssertionError:
# print(song_idx, song_length)
song_feats.append(cnn_forward(torch.tensor([[data]], device=torch.device('cpu'))))
# print("song feats", song_feats.__len__(), song_feats[0].shape)
>>>>>>> Stashed changes
return torch.cat(song_feats)
h5data, idx_list, x_lengths_list, labels_list = batch
......@@ -97,7 +124,12 @@ class CRNN(BasePtlModel):
for n, ind in enumerate(idx_list):
sequences.append(extract_features(ind, x_lengths_list[n]))
<<<<<<< Updated upstream
sequences_padded = torch.nn.utils.rnn.pad_sequence(sequences, batch_first=True)
=======
# print("sequences", sequences.__len__(), sequences[0].shape)
sequences_padded = torch.nn.utils.rnn.pad_sequence(sequences, batch_first=False)
>>>>>>> Stashed changes
result = rnn_forward(sequences_padded)
return result
......@@ -145,14 +177,14 @@ class CRNN(BasePtlModel):
def training_step(self, data_batch, batch_i):
y = data_batch[-1]
y_hat = self.forward(data_batch)
y = y.float()
y = torch.stack(y).float()
y_hat = y_hat.float()
return {'loss': self.loss(y_hat, y)}
def validation_step(self, data_batch, batch_i):
y = data_batch[-1]
y_hat = self.forward(data_batch)
y = y.float()
y = torch.stack(y).float()
y_hat = y_hat.float()
return {
'val_loss': self.loss(y_hat, y),
......@@ -163,7 +195,7 @@ class CRNN(BasePtlModel):
def test_step(self, data_batch, batch_i):
y = data_batch[-1]
y_hat = self.forward(data_batch)
y = y.float()
y = torch.stack(y).float()
y_hat = y_hat.float()
return {
'test_loss': self.loss(y_hat, y),
......@@ -180,12 +212,18 @@ class CRNN(BasePtlModel):
# network params
parser.add_argument('--gru_hidden_size', default=320, type=int)
parser.add_argument('--gru_num_layers', default=2, type=int)
parser.opt_list('--drop_prob', default=0.2, options=[0.2, 0.5], type=float, tunable=True)
parser.opt_list('--drop_prob', default=0.2, options=[0.2, 0.5], type=float, tunable=False)
parser.opt_list('--learning_rate', default=0.0001, type=float,
options=[0.00001, 0.0005, 0.001],
<<<<<<< Updated upstream
tunable=True)
parser.opt_list('--slicing_mode', default='full', options=['full', 'slice'], type=str, tunable=False)
parser.opt_list('--input_size', default=512, options=[512, 1024], type=int, tunable=True)
=======
tunable=False)
parser.opt_list('--slicing_mode', default='full', options=['full', 'slice'], type=str, tunable=True)
parser.opt_list('--input_size', default=512, options=[512, 1024], type=int, tunable=False)
>>>>>>> Stashed changes
# training params (opt)
......@@ -194,7 +232,7 @@ class CRNN(BasePtlModel):
# if using 2 nodes with 4 gpus each the batch size here
# (256) will be 256 / (2*8) = 16 per gpu
parser.opt_list('--batch_size', default=32, type=int,
options=[16, 32], tunable=False,
parser.opt_list('--batch_size', default=16, type=int,
options=[16, 8], tunable=True,
help='batch size will be divided over all gpus being used across all nodes')
return parser
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment