Seriously, no idea. Wild amount of changes. Good luck.

python/save_inv_data_full.py

@@ -1,24 +1,42 @@
-import glob
 import numpy as np
 import matplotlib.pyplot as plt
 from thunderhopper.modeltools import load_data, save_data
-from thunderhopper.filetools import crop_paths
+from thunderhopper.filetools import search_files, crop_paths
 from thunderhopper.filtertools import find_kern_specs
-from thunderhopper.model import process_signal, convolve_kernels
+from thunderhopper.model import process_signal
+from misc_functions import draw_noise_segment
 from IPython import embed
 
 # GENERAL SETTINGS:
-target = 'Omocestus_rufipes'
-data_paths = glob.glob(f'../data/processed/{target}*.npz')
+target_species = [
+    'Chorthippus_biguttulus',
+    'Chorthippus_mollis',
+    'Chrysochraon_dispar',
+    'Euchorthippus_declivus',
+    'Gomphocerippus_rufus',
+    'Omocestus_rufipes',
+    'Pseudochorthippus_parallelus',
+][0]
+example_file = {
+    'Chorthippus_biguttulus': 'Chorthippus_biguttulus_GBC_94-17s73.1ms-19s977ms',
+    'Chorthippus_mollis': 'Chorthippus_mollis_DJN_41_T28C-46s4.58ms-1m15s697ms',
+    'Chrysochraon_dispar': 'Chrysochraon_dispar_DJN_26_T28C_DT-32s134ms-34s432ms',
+    'Euchorthippus_declivus': 'Euchorthippus_declivus_FTN_79-2s167ms-2s563ms',
+    'Gomphocerippus_rufus': 'Gomphocerippus_rufus_FTN_91-3-884ms-10s427ms',
+    'Omocestus_rufipes': 'Omocestus_rufipes_DJN_32-40s724ms-48s779ms',
+    'Pseudochorthippus_parallelus': 'Pseudochorthippus_parallelus_GBC_88-6s678ms-9s32.3ms'
+}[target_species]
+data_paths = search_files(target_species, dir='../data/processed/')
 noise_path = '../data/processed/white_noise_sd-1.npz'
+ref_path = '../data/inv/full/ref_measures.npz'
 stages = ['filt', 'env', 'log', 'inv', 'conv', 'feat']
 save_path = '../data/inv/full/'
 
 # ANALYSIS SETTINGS:
 example_scales = np.array([0.1, 1, 10, 30, 100, 300])
-scales = np.geomspace(0.01, 10000, 100)
-scales = np.unique(np.concatenate((scales, example_scales)))
-thresh_rel = 3
+scales = np.geomspace(0.01, 10000, 500)
+scales = np.unique(np.concatenate(([0], scales, example_scales)))
+thresh_rel = 0.5
 
 # SUBSET SETTINGS:
 kernels = np.array([
@@ -34,11 +52,14 @@ types = None#np.array([-1])
 sigmas = None#np.array([0.001, 0.002, 0.004, 0.008, 0.016, 0.032])
 
 # PREPARATION:
-noise_data = np.load(noise_path)
-pure_noise = noise_data['raw']
+pure_noise = np.load(noise_path)['raw']
+if thresh_rel is not None:
+    # Get threshold values from pure-noise response SD:
+    thresh_abs = np.load(ref_path)['conv'] * thresh_rel
 
 # EXECUTION:
 for data_path, name in zip(data_paths, crop_paths(data_paths)):
+    save_detailed = example_file in name
     print(f'Processing {name}')
 
     # Get song recording (prior to anything):
@@ -46,8 +67,8 @@ for data_path, name in zip(data_paths, crop_paths(data_paths)):
     song, rate = data['raw'], config['rate']
 
     if thresh_rel is not None:
-        # Get noise-bound kernel-specific thresholds:
-        config['feat_thresh'] = noise_data['conv'].std(axis=0) * thresh_rel
+        # Set kernel-specific thresholds:
+        config['feat_thresh'] = thresh_abs
 
     # Reduce to kernel subset:
     if any(var is not None for var in [kernels, types, sigmas]):
@@ -66,22 +87,10 @@ for data_path, name in zip(data_paths, crop_paths(data_paths)):
     song /= song[segment].std(axis=0)
 
     # Get normalized noise component:
-    noise = pure_noise[:song.shape[0]]
+    noise = draw_noise_segment(pure_noise, song.shape[0])
     noise /= noise[segment].std()
 
-    # Prepare snippet storage:
-    shape_low = (song.shape[0], example_scales.size)
-    shape_high = (song.shape[0], config['k_specs'].shape[0], example_scales.size)
-    snippets = dict(
-        snip_filt=np.zeros(shape_low, dtype=float),
-        snip_env=np.zeros(shape_low, dtype=float),
-        snip_log=np.zeros(shape_low, dtype=float),
-        snip_inv=np.zeros(shape_low, dtype=float),
-        snip_conv=np.zeros(shape_high, dtype=float),
-        snip_feat=np.zeros(shape_high, dtype=float)
-    )
-
-    # Prepare measure storage:
+    # Prepare storage:
     shape_low = (scales.size,)
     shape_high = (scales.size, config['k_specs'].shape[0])
     measures = dict(
@@ -91,6 +100,18 @@ for data_path, name in zip(data_paths, crop_paths(data_paths)):
         measure_inv=np.zeros(shape_low, dtype=float),
         measure_conv=np.zeros(shape_high, dtype=float),
         measure_feat=np.zeros(shape_high, dtype=float)
+    ) 
+    if save_detailed:
+        # Prepare optional storage:
+        shape_low = (song.shape[0], example_scales.size)
+        shape_high = (song.shape[0], config['k_specs'].shape[0], example_scales.size)
+        snippets = dict(
+            snip_filt=np.zeros(shape_low, dtype=float),
+            snip_env=np.zeros(shape_low, dtype=float),
+            snip_log=np.zeros(shape_low, dtype=float),
+            snip_inv=np.zeros(shape_low, dtype=float),
+            snip_conv=np.zeros(shape_high, dtype=float),
+            snip_feat=np.zeros(shape_high, dtype=float)
     )
 
     # Execute piecewise:
@@ -105,18 +126,17 @@ for data_path, name in zip(data_paths, crop_paths(data_paths)):
                                         signal=scaled, rate=rate)
         # Store results:
         for stage in stages:
-            mkey, skey = f'measure_{stage}', f'snip_{stage}'
-
-            # Log snippet data:
-            if scale in example_scales:
-                scale_ind = np.nonzero(example_scales == scale)[0][0]
-                snippets[skey][:, ..., scale_ind] = signals[stage]
-
-            # Log intensity measure per stage (excluding binary):
-            if stage in ['raw', 'filt', 'env', 'log', 'inv', 'conv']:
-                measures[mkey][i] = signals[stage][segment, ...].std(axis=0)
-            elif stage == 'feat':
+            # Log intensity measures:
+            mkey = f'measure_{stage}'
+            if stage == 'feat':
                 measures[mkey][i] = signals[stage][segment, :].mean(axis=0)
+            else:
+                measures[mkey][i] = signals[stage][segment, ...].std(axis=0)
+
+            # Log optional snippet data:
+            if save_detailed and scale in example_scales:
+                scale_ind = np.nonzero(example_scales == scale)[0][0]
+                snippets[f'snip_{stage}'][:, ..., scale_ind] = signals[stage]
 
     # Save analysis results:
     if save_path is not None:
@@ -124,8 +144,9 @@ for data_path, name in zip(data_paths, crop_paths(data_paths)):
             scales=scales,
             example_scales=example_scales,
             )
-        data.update(snippets)
         data.update(measures)
+        if save_detailed:
+            data.update(snippets)
         save_data(save_path + name, data, config, overwrite=True)
 print('Done.')
 embed()