Commit 01e50a64 authored by Shreyan Chowdhury's avatar Shreyan Chowdhury

refactor utils.py

parent 081ed833
......@@ -19,28 +19,29 @@ plt.rcParams["figure.dpi"] = 288 # increase dpi for clearer plots
# PARAMS =======================
INPUT_SIZE = (96, 256)
MAX_LENGTH = 10000
data_roots = {
"mtgjamendo":{
"rechenknecht3.cp.jku.at": "/media/rk3/shared/datasets/MTG-Jamendo",
"rechenknecht2.cp.jku.at": "/media/rk2/shared/datasets/MTG-Jamendo",
"rechenknecht1.cp.jku.at": "/media/rk1/shared/datasets/MTG-Jamendo",
"hermine": "/media/verena/SAMSUNG/Data/MTG-Jamendo",
"verena-830g5": "/media/verena/SAMSUNG/Data/MTG-Jamendo",
"shreyan-HP": "/home/shreyan/mounts/home@rk3/shared/datasets/MTG-Jamendo",
"shreyan-All-Series": "/mnt/2tb/datasets/MTG-Jamendo"
},
"midlevel":{
"rechenknecht3.cp.jku.at": "/media/rk3/shared/midlevel",
"rechenknecht2.cp.jku.at": "/media/rk2/shared/datasets/midlevel",
"rechenknecht1.cp.jku.at": "/media/rk1/shared/datasets/midlevel",
"hermine":"",
"verena-830g5":"",
"shreyan-HP": "/mnt/2tb/datasets/MidlevelFeatures",
"shreyan-All-Series": "/mnt/2tb/datasets/MidlevelFeatures"
}
}
use_dataset= "mtgjamendo"
# data_roots = {
# "mtgjamendo":{
# "rechenknecht5.cp.jku.at": "/media/rk5/shared/datasets/MTG-Jamendo",
# "rechenknecht3.cp.jku.at": "/media/rk3/shared/datasets/MTG-Jamendo",
# "rechenknecht2.cp.jku.at": "/media/rk2/shared/datasets/MTG-Jamendo",
# "rechenknecht1.cp.jku.at": "/media/rk1/shared/datasets/MTG-Jamendo",
# "hermine": "/media/verena/SAMSUNG/Data/MTG-Jamendo",
# "verena-830g5": "/media/verena/SAMSUNG/Data/MTG-Jamendo",
# "shreyan-HP": "/home/shreyan/mounts/home@rk3/shared/datasets/MTG-Jamendo",
# "shreyan-All-Series": "/mnt/2tb/datasets/MTG-Jamendo"
# },
# "midlevel":{
# "rechenknecht3.cp.jku.at": "/media/rk3/shared/midlevel",
# "rechenknecht2.cp.jku.at": "/media/rk2/shared/datasets/midlevel",
# "rechenknecht1.cp.jku.at": "/media/rk1/shared/datasets/midlevel",
# "hermine":"",
# "verena-830g5":"",
# "shreyan-HP": "/mnt/2tb/datasets/MidlevelFeatures",
# "shreyan-All-Series": "/mnt/2tb/datasets/MidlevelFeatures"
# }
# }
#
# use_dataset= "mtgjamendo"
# CONFIG =======================
# paths:
......@@ -51,83 +52,65 @@ PATH_RESULTS = os.path.join(PATH_PROJECT_ROOT, 'results')
hostname = os.uname()[1]
username = getpass.getuser()
if hostname in ['rechenknecht3.cp.jku.at']:
plt.switch_backend('agg')
PATH_DATA_CACHE = '/media/rk3/shared/kofta_cached_datasets'
USE_GPU = True
elif hostname == 'rechenknecht2.cp.jku.at':
plt.switch_backend('agg')
PATH_DATA_CACHE = '/media/rk2/shared/kofta_cached_datasets'
USE_GPU = True
elif hostname == 'rechenknecht1.cp.jku.at':
plt.switch_backend('agg')
PATH_DATA_CACHE = '/media/rk1/shared/kofta_cached_datasets'
USE_GPU = True
elif hostname == 'hermine': # PC verena
plt.switch_backend('agg')
USE_GPU = True
elif hostname == 'verena-830g5': # Laptop Verena
if hostname in ['verena-830g5', 'shreyan-HP', 'shreyan-All-Series']:
USE_GPU = False
elif hostname == 'shreyan-HP': # Laptop Shreyan
USE_GPU = False
PATH_DATA_CACHE = '/home/shreyan/mounts/home@rk2/shared/kofta_cached_datasets'
else:
# PATH_DATA_CACHE = '/home/shreyan/mounts/home@rk3/shared/kofta_cached_datasets'
PATH_DATA_CACHE = '/mnt/2tb/datasets/data_caches'
USE_GPU = False
USE_GPU = True
if username == 'verena':
PATH_RESULTS = '/home/verena/experiments/moodwalk/'
PATH_DATA_ROOT = data_roots[use_dataset][hostname]
PATH_AUDIO = os.path.join(PATH_DATA_ROOT, 'MTG-Jamendo_audio')
PATH_ANNOTATIONS = os.path.join(PATH_DATA_ROOT, 'MTG-Jamendo_annotations')
PATH_MELSPEC_DOWNLOADED = os.path.join(PATH_DATA_ROOT, 'MTG-Jamendo_melspec_downloaded')
PATH_MELSPEC_DOWNLOADED_FRAMED = os.path.join(PATH_MELSPEC_DOWNLOADED, 'framed')
PATH_MELSPEC_DOWNLOADED_HDF5 = os.path.join(PATH_DATA_ROOT, 'HDF5Cache_spectrograms')
TRAINED_MODELS_PATH = ''
def set_paths(dataset_name):
global PATH_DATA_ROOT, PATH_AUDIO, PATH_ANNOTATIONS
PATH_DATA_ROOT = data_roots[dataset_name][hostname]
if dataset_name=='midlevel':
PATH_AUDIO = os.path.join(PATH_DATA_ROOT, 'audio')
PATH_ANNOTATIONS = os.path.join(PATH_DATA_ROOT, 'metadata_annotations')
elif dataset_name=='mtgjamendo':
PATH_AUDIO = os.path.join(PATH_DATA_ROOT, 'MTG-Jamendo_audio')
PATH_ANNOTATIONS = os.path.join(PATH_DATA_ROOT, 'MTG-Jamendo_annotations')
def get_paths(dataset_name):
PATH_DATA_ROOT = data_roots[dataset_name][hostname]
if dataset_name == 'midlevel':
PATH_AUDIO = os.path.join(PATH_DATA_ROOT, 'audio')
PATH_ANNOTATIONS = os.path.join(PATH_DATA_ROOT, 'metadata_annotations')
elif dataset_name == 'mtgjamendo':
PATH_AUDIO = os.path.join(PATH_DATA_ROOT, 'MTG-Jamendo_audio')
PATH_ANNOTATIONS = os.path.join(PATH_DATA_ROOT, 'MTG-Jamendo_annotations')
else:
PATH_AUDIO = os.path.join(PATH_DATA_ROOT, 'audio')
PATH_ANNOTATIONS = os.path.join(PATH_DATA_ROOT, 'annotations')
return PATH_DATA_ROOT, PATH_AUDIO, PATH_ANNOTATIONS
# PATH_DATA_ROOT = data_roots[use_dataset][hostname]
#
# PATH_AUDIO = os.path.join(PATH_DATA_ROOT, 'MTG-Jamendo_audio')
# PATH_ANNOTATIONS = os.path.join(PATH_DATA_ROOT, 'MTG-Jamendo_annotations')
# PATH_MELSPEC_DOWNLOADED = os.path.join(PATH_DATA_ROOT, 'MTG-Jamendo_melspec_downloaded')
# PATH_MELSPEC_DOWNLOADED_FRAMED = os.path.join(PATH_MELSPEC_DOWNLOADED, 'framed')
# PATH_MELSPEC_DOWNLOADED_HDF5 = os.path.join(PATH_DATA_ROOT, 'HDF5Cache_spectrograms')
# TRAINED_MODELS_PATH = ''
#
# def set_paths(dataset_name):
# global PATH_DATA_ROOT, PATH_AUDIO, PATH_ANNOTATIONS
# PATH_DATA_ROOT = data_roots[dataset_name][hostname]
# if dataset_name=='midlevel':
# PATH_AUDIO = os.path.join(PATH_DATA_ROOT, 'audio')
# PATH_ANNOTATIONS = os.path.join(PATH_DATA_ROOT, 'metadata_annotations')
# elif dataset_name=='mtgjamendo':
# PATH_AUDIO = os.path.join(PATH_DATA_ROOT, 'MTG-Jamendo_audio')
# PATH_ANNOTATIONS = os.path.join(PATH_DATA_ROOT, 'MTG-Jamendo_annotations')
# def get_paths(dataset_name):
# PATH_DATA_ROOT = data_roots[dataset_name][hostname]
# if dataset_name == 'midlevel':
# PATH_AUDIO = os.path.join(PATH_DATA_ROOT, 'audio')
# PATH_ANNOTATIONS = os.path.join(PATH_DATA_ROOT, 'metadata_annotations')
# elif dataset_name == 'mtgjamendo':
# PATH_AUDIO = os.path.join(PATH_DATA_ROOT, 'MTG-Jamendo_audio')
# PATH_ANNOTATIONS = os.path.join(PATH_DATA_ROOT, 'MTG-Jamendo_annotations')
# else:
# PATH_AUDIO = os.path.join(PATH_DATA_ROOT, 'audio')
# PATH_ANNOTATIONS = os.path.join(PATH_DATA_ROOT, 'annotations')
# return PATH_DATA_ROOT, PATH_AUDIO, PATH_ANNOTATIONS
# run name
def make_run_name(suffix=''):
def make_run_name(suffix=None):
# assert ' ' not in suffix
hash = hashlib.sha1()
hash.update(str(time.time()).encode('utf-8'))
run_hash = hash.hexdigest()[:5]
name = run_hash + f' - {suffix}'
if suffix is not None:
name = run_hash + f' - {suffix}'
else:
name = run_hash
return name
CURR_RUN_PATH = ''
curr_run_name = ''
filelog = logging.getLogger()
streamlog = logging.getLogger()
logger = logging.getLogger()
filelog = logging.getLogger('mw_file')
streamlog = logging.getLogger('mw_stream')
logger = logging.getLogger('mw_log')
def init_experiment(comment='', name=None):
global CURR_RUN_PATH, curr_run_name
......@@ -136,7 +119,8 @@ def init_experiment(comment='', name=None):
curr_run_name = make_run_name(comment)
else:
curr_run_name = name
CURR_RUN_PATH = os.path.join(PATH_RESULTS, 'runs', 'running - '+curr_run_name)
CURR_RUN_PATH = os.path.join(PATH_RESULTS, 'runs', curr_run_name)
# CURR_RUN_PATH = os.path.join(PATH_RESULTS, 'runs', 'running - '+curr_run_name)
if not os.path.isdir(CURR_RUN_PATH):
os.makedirs(CURR_RUN_PATH)
......@@ -163,6 +147,8 @@ def init_experiment(comment='', name=None):
# ============================================
return CURR_RUN_PATH, logger, filelog, streamlog
def exit_experiment(status, exp):
global CURR_RUN_PATH
if status=='failed':
......@@ -184,6 +170,22 @@ def write_to_file(data, path):
f.writelines(i)
def dict_to_entry(dict, roundoff=4, filter=None, exclude=False):
if filter is None:
filter = dict.keys()
exclude = False
dict = {k:dict.get(k) for k in list(set(dict.keys()) - set(filter))} if exclude else {k:dict.get(k) for k in filter}
if roundoff is not None:
for key in dict.keys():
try:
dict[key] = np.round(dict[key], roundoff)
except:
continue
return str(dict)[1:-1]
def dims_calc(obj, in_shape):
"""
utility function to calculate output dimensions of a conv2d or maxpool2d stage
......@@ -227,10 +229,11 @@ def save(model, path):
if __name__=='__main__':
# TESTS
c = nn.Conv2d(1, 64, 3, 1, 1) # (in_channels, out_channels, kernel_size, stride, padding)
# m = nn.MaxPool2d(2)
print(dims_calc(c, [256, 600, 1]))
d = {'corr_avg': 0.47045496, 'pval_avg': 9.22310376114139e-09, 'avg_test_loss': torch.tensor(1.5252, device='cpu')}
print(dict_to_entry(d, 2))
# c = nn.Conv2d(1, 64, 3, 1, 1) # (in_channels, out_channels, kernel_size, stride, padding)
# # m = nn.MaxPool2d(2)
# print(dims_calc(c, [256, 600, 1]))
# preprocess_specs(source_root=PATH_MELSPEC_DOWNLOADED,
# destination_root=PATH_MELSPEC_DOWNLOADED_FRAMED)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment