run status, full song

experiments/experiment_crnn.py

-from utils import USE_GPU, init_experiment
+from utils import USE_GPU, init_experiment, exit_experiment
 from pytorch_lightning import Trainer
 from pytorch_lightning.callbacks import EarlyStopping, ModelCheckpoint
 from test_tube import Experiment, HyperOptArgumentParser
@@ -60,13 +60,29 @@ def run(hparams):
                           nb_sanity_val_steps=0)  # don't run sanity validation run
     else:
         trainer = Trainer(experiment=exp, max_nb_epochs=1, train_percent_check=0.1,
-                          fast_dev_run=True)
+                          fast_dev_run=True, nb_sanity_val_steps=0)
 
     model = Network(num_class=56, config=model_config, hparams=hparams)
 
     print(model)
-    trainer.fit(model)
-    trainer.test()
+
+    try:
+        trainer.fit(model)
+    except KeyboardInterrupt:
+        logger.info("Training interrupted")
+    except:
+        logger.exception(msg="Error occurred during train!")
+        exit_experiment('failed', exp)
+
+    try:
+        logger.info("Starting test...")
+        trainer.test()
+    except KeyboardInterrupt:
+        logger.info("Exiting...")
+        exit_experiment('stopped', exp)
+    except:
+        logger.exception(msg="Error occurred during test!")
+        exit_experiment('failed', exp)
 
 
 if __name__ == '__main__':

models/crnn.py

@@ -71,16 +71,26 @@ class CRNN(BasePtlModel):
 
             # classifier
             x = x.view(-1, x.size(0), 32)
+<<<<<<< Updated upstream
+=======
+            # print(x.shape)
+>>>>>>> Stashed changes
 
             return x
 
         def rnn_forward(x):
+<<<<<<< Updated upstream
+=======
+            # print(x.squeeze().shape)
+            x = x.squeeze()
+>>>>>>> Stashed changes
             x = self.gru1(x)[1][1]  # TODO: Check if this is correct
             x = self.dropout(x)
             logit = nn.Sigmoid()(self.dense(x))
             return logit
 
         def extract_features(song_idx, song_length):
+<<<<<<< Updated upstream
             song_feats = []
             for l in range(song_length//self.input_size + 1):
                 data = h5data[song_idx + l*self.input_size:song_idx + min(song_length, (l + 1) * self.input_size)].transpose()
@@ -90,6 +100,23 @@ class CRNN(BasePtlModel):
                 except:
                     song_feats.append(cnn_forward(torch.tensor([[data]], device=torch.device('cpu'))))
 
+=======
+            # print(song_idx, song_length)
+            song_length = 2560
+            song_feats = []
+            for l in range(song_length//self.input_size + 1):
+                data = h5data[song_idx + l*self.input_size:song_idx + min(song_length, (l + 1) * self.input_size)].transpose()
+                if data.shape[1] < self.input_size*0.25:
+                    continue
+                data = np.pad(data, ((0, 0), (0, self.input_size-data.shape[1])), mode='wrap')
+                try:
+                    song_feats.append(cnn_forward(torch.tensor([[data]], device=torch.device('cuda'))))
+                except AssertionError:
+                    # print(song_idx, song_length)
+                    song_feats.append(cnn_forward(torch.tensor([[data]], device=torch.device('cpu'))))
+
+            # print("song feats", song_feats.__len__(), song_feats[0].shape)
+>>>>>>> Stashed changes
             return torch.cat(song_feats)
 
         h5data, idx_list, x_lengths_list, labels_list = batch
@@ -97,7 +124,12 @@ class CRNN(BasePtlModel):
         for n, ind in enumerate(idx_list):
             sequences.append(extract_features(ind, x_lengths_list[n]))
 
+<<<<<<< Updated upstream
         sequences_padded = torch.nn.utils.rnn.pad_sequence(sequences, batch_first=True)
+=======
+        # print("sequences", sequences.__len__(), sequences[0].shape)
+        sequences_padded = torch.nn.utils.rnn.pad_sequence(sequences, batch_first=False)
+>>>>>>> Stashed changes
         result = rnn_forward(sequences_padded)
         return result
 
@@ -145,14 +177,14 @@ class CRNN(BasePtlModel):
     def training_step(self, data_batch, batch_i):
         y = data_batch[-1]
         y_hat = self.forward(data_batch)
-        y = y.float()
+        y = torch.stack(y).float()
         y_hat = y_hat.float()
         return {'loss': self.loss(y_hat, y)}
 
     def validation_step(self, data_batch, batch_i):
         y = data_batch[-1]
         y_hat = self.forward(data_batch)
-        y = y.float()
+        y = torch.stack(y).float()
         y_hat = y_hat.float()
         return {
             'val_loss': self.loss(y_hat, y),
@@ -163,7 +195,7 @@ class CRNN(BasePtlModel):
     def test_step(self, data_batch, batch_i):
         y = data_batch[-1]
         y_hat = self.forward(data_batch)
-        y = y.float()
+        y = torch.stack(y).float()
         y_hat = y_hat.float()
         return {
             'test_loss': self.loss(y_hat, y),
@@ -180,12 +212,18 @@ class CRNN(BasePtlModel):
         # network params
         parser.add_argument('--gru_hidden_size', default=320, type=int)
         parser.add_argument('--gru_num_layers', default=2, type=int)
-        parser.opt_list('--drop_prob', default=0.2, options=[0.2, 0.5], type=float, tunable=True)
+        parser.opt_list('--drop_prob', default=0.2, options=[0.2, 0.5], type=float, tunable=False)
         parser.opt_list('--learning_rate', default=0.0001, type=float,
                         options=[0.00001, 0.0005, 0.001],
+<<<<<<< Updated upstream
                         tunable=True)
         parser.opt_list('--slicing_mode', default='full', options=['full', 'slice'], type=str, tunable=False)
         parser.opt_list('--input_size', default=512, options=[512, 1024], type=int, tunable=True)
+=======
+                        tunable=False)
+        parser.opt_list('--slicing_mode', default='full', options=['full', 'slice'], type=str, tunable=True)
+        parser.opt_list('--input_size', default=512, options=[512, 1024], type=int, tunable=False)
+>>>>>>> Stashed changes
 
 
         # training params (opt)
@@ -194,7 +232,7 @@ class CRNN(BasePtlModel):
 
         # if using 2 nodes with 4 gpus each the batch size here
         #  (256) will be 256 / (2*8) = 16 per gpu
-        parser.opt_list('--batch_size', default=32, type=int,
-                        options=[16, 32], tunable=False,
+        parser.opt_list('--batch_size', default=16, type=int,
+                        options=[16, 8], tunable=True,
                         help='batch size will be divided over all gpus being used across all nodes')
         return parser