Commit 081ed833 authored by Shreyan Chowdhury's avatar Shreyan Chowdhury

add midlevel experiments

parent 1fbb3082
import logging
from pytorch_lightning import Trainer
from pytorch_lightning.callbacks import EarlyStopping, ModelCheckpoint
from test_tube import Experiment, HyperOptArgumentParser
import os
from models.cp_resnet_locpad import Network
from utils import init_experiment, USE_GPU, dict_to_entry
initialized = False # TODO: Find a better way to do this
trial_counter = 0
def main(hparams, gpu=None):
if gpu is None:
gpu = [hparams.gpu]
global initialized, trial_counter
trial_counter += 1
if not initialized:
init_experiment(comment=hparams.name)
from utils import CURR_RUN_PATH # import these after init_experiment
logger = logging.getLogger(__name__)
if not initialized:
logger.info(f"tensorboard --logdir={CURR_RUN_PATH}")
initialized = True
trial_name = f"trial_{trial_counter}"
logger.info(trial_name)
hpdict = hparams.__dict__.copy()
for key in ['trials', 'hpc_exp_number', 'optimize_trials_parallel_gpu', 'generate_trials', 'optimize_parallel_cpu',
'optimize_parallel_gpu', 'optimize_parallel', 'load_from_trial_name']:
if key in hpdict: del hpdict[key]
logger.info(dict_to_entry(hpdict))
exp = Experiment(name=trial_name, save_dir=CURR_RUN_PATH)
model_config = {
'validation_metrics': ['corr_avg'],
'test_metrics': ['corr_avg', 'corr']
}
# build model
model = Network(config=model_config, hparams=hparams, num_targets=7)
# callbacks
early_stop = EarlyStopping(
monitor='corr_avg',
patience=20,
verbose=True,
mode='max'
)
model_save_path = os.path.join(CURR_RUN_PATH, trial_name, 'best.ckpt')
# model_save_path = os.path.join(CURR_RUN_PATH, exp.name)
checkpoint = ModelCheckpoint(
filepath=model_save_path,
save_best_only=True,
verbose=True,
monitor='corr_avg',
mode='max'
)
# configure trainer
if USE_GPU:
trainer = Trainer(gpus=gpu, distributed_backend=None,
experiment=exp, max_nb_epochs=hparams.epochs,
train_percent_check=hparams.train_percent,
fast_dev_run=False, early_stop_callback=early_stop,
checkpoint_callback=checkpoint,
nb_sanity_val_steps=0) # don't run sanity validation run
else:
trainer = Trainer(experiment=exp, max_nb_epochs=1, train_percent_check=0.01, val_percent_check=0.01,
fast_dev_run=True, nb_sanity_val_steps=0)
if hparams.mode == 'train':
# train model
exp.tag(hpdict)
trainer.fit(model)
if hparams.mode == 'test':
from utils import PATH_RESULTS
exp_dir = os.path.join(PATH_RESULTS, 'runs', hparams.name)
ckpt_dir = os.path.join(exp_dir, hparams.trial_name, 'best.ckpt')
ckpt_files = os.listdir(ckpt_dir)
if len(ckpt_files) == 1:
weights_file = os.path.join(ckpt_dir, ckpt_files[0])
else:
logger.warning(f"Multiple weight files, picking {ckpt_files[-1]}")
weights_file = os.path.join(ckpt_dir, ckpt_files[-1])
model = Network.load_from_metrics(
weights_path=weights_file,
tags_csv=os.path.join(exp_dir, trial_name, 'version_0/meta_tags.csv'),
on_gpu=True,
config=model_config
)
trainer.test(model=model)
if hparams.mode == 'train and test':
exp.tag(hpdict)
trainer.fit(model)
trainer.test()
if __name__=='__main__':
parent_parser = HyperOptArgumentParser(strategy='grid_search', add_help=False)
parent_parser.add_argument('--gpu', type=int,
default=0, help='which gpu to use')
parent_parser.add_argument('--name', type=str,
default='pt_lightning_exp_a', help='test tube exp name')
parent_parser.add_argument('--train_percent', type=float,
default=1.0, help='how much train data to use')
parent_parser.add_argument('--epochs', type=int,
default=10, help='maximum number of epochs')
parent_parser.add_argument('--mode', type=str,
default='train and test', help='train/test/train and test')
parent_parser.add_argument('--load_from_trial_name', type=str,
default='trial_0', help='which trial name to load a model from')
parser = Network.add_model_specific_args(parent_parser)
hyperparams = parser.parse_args()
# hyperparams.optimize_parallel_gpu(main, gpu_ids=['0','1','2','3'])
for hparams in hyperparams.trials(18):
main(hparams)
import logging
from pytorch_lightning import Trainer
from pytorch_lightning.callbacks import EarlyStopping, ModelCheckpoint
from test_tube import Experiment, HyperOptArgumentParser
import os
from models.midlevel_vgg import ModelMidlevelBasic as Network
from utils import init_experiment, USE_GPU, dict_to_entry
initialized = False # TODO: Find a better way to do this
trial_counter = 0
def main(hparams, gpu=None):
if gpu is None:
gpu = [hparams.gpu]
global initialized, trial_counter
trial_counter += 1
if not initialized:
init_experiment(comment=hparams.name)
from utils import CURR_RUN_PATH # import these after init_experiment
logger = logging.getLogger('mw_log')
if not initialized:
logger.info(f"tensorboard --logdir={CURR_RUN_PATH}")
initialized = True
trial_name = f"trial_{trial_counter}"
logger.info(trial_name)
hpdict = hparams.__dict__.copy()
for key in ['trials', 'hpc_exp_number', 'optimize_trials_parallel_gpu', 'generate_trials', 'optimize_parallel_cpu',
'optimize_parallel_gpu', 'optimize_parallel', 'load_from_trial_name']:
if key in hpdict: del hpdict[key]
logger.info(dict_to_entry(hpdict, roundoff=None))
exp = Experiment(name=trial_name, save_dir=CURR_RUN_PATH)
model_config = {
'validation_metrics': ['corr_avg'],
'test_metrics': ['corr_avg', 'corr']
}
# build model
model = Network(config=model_config, hparams=hparams, num_targets=7)
# callbacks
early_stop = EarlyStopping(
monitor='corr_avg',
patience=20,
verbose=True,
mode='max'
)
model_save_path = os.path.join(CURR_RUN_PATH, trial_name, 'best.ckpt')
# model_save_path = os.path.join(CURR_RUN_PATH, exp.name)
checkpoint = ModelCheckpoint(
filepath=model_save_path,
save_best_only=True,
verbose=True,
monitor='corr_avg',
mode='max'
)
# configure trainer
if USE_GPU:
trainer = Trainer(gpus=gpu, distributed_backend=None,
experiment=exp, max_nb_epochs=hparams.epochs,
train_percent_check=hparams.train_percent,
fast_dev_run=False, early_stop_callback=early_stop,
checkpoint_callback=checkpoint,
nb_sanity_val_steps=0) # don't run sanity validation run
else:
trainer = Trainer(experiment=exp, max_nb_epochs=1, train_percent_check=0.01, val_percent_check=0.01,
fast_dev_run=True, nb_sanity_val_steps=0)
if hparams.mode == 'train':
# train model
exp.tag(hpdict)
trainer.fit(model)
if hparams.mode == 'test':
from utils import PATH_RESULTS
exp_dir = os.path.join(PATH_RESULTS, 'runs', hparams.name)
ckpt_dir = os.path.join(exp_dir, hparams.trial_name, 'best.ckpt')
ckpt_files = os.listdir(ckpt_dir)
if len(ckpt_files) == 1:
weights_file = os.path.join(ckpt_dir, ckpt_files[0])
else:
logger.warning(f"Multiple weight files, picking {ckpt_files[-1]}")
weights_file = os.path.join(ckpt_dir, ckpt_files[-1])
model = Network.load_from_metrics(
weights_path=weights_file,
tags_csv=os.path.join(exp_dir, trial_name, 'version_0/meta_tags.csv'),
on_gpu=True,
config=model_config
)
trainer.test(model=model)
if hparams.mode == 'train and test':
exp.tag(hpdict)
trainer.fit(model)
trainer.test()
if __name__=='__main__':
parent_parser = HyperOptArgumentParser(strategy='grid_search', add_help=False)
parent_parser.add_argument('--gpu', type=int,
default=0, help='which gpu to use')
parent_parser.add_argument('--name', type=str,
default='pt_lightning_exp_a', help='test tube exp name')
parent_parser.add_argument('--train_percent', type=float,
default=1.0, help='how much train data to use')
parent_parser.add_argument('--epochs', type=int,
default=10, help='maximum number of epochs')
parent_parser.add_argument('--mode', type=str,
default='train and test', help='train/test/train and test')
parent_parser.add_argument('--load_from_trial_name', type=str,
default='trial_0', help='which trial name to load a model from')
parser = Network.add_model_specific_args(parent_parser)
hyperparams = parser.parse_args()
# hyperparams.optimize_parallel_gpu(main, gpu_ids=['0','1','2','3'])
for hparams in hyperparams.trials(18):
main(hparams)
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment