Commit f7152d31 authored by Paulo Medeiros's avatar Paulo Medeiros
Browse files

Group metrics options in their own config section

parent 5a7e406a
......@@ -46,14 +46,20 @@
# Note that the interval is closed at the left and open at the right.
dtgs.cycle_length = '3H' # Default = '3H'
#
# custom_metrics_optimize_mode:
[metrics]
# method:
# > default: "haversine"
# > choices: "correlation", "haversine"
method = "correlation"
# optimize_mode:
# > default: "memory"
# > choices: "memory", "speed_mem_compromise", "speed"
# In terms of memory usage:
# "memory" < "speed_mem_compromise" < "speed"
# In terms of execution time:
# "memory" > "speed_mem_compromise" > "speed"
custom_metrics_optimize_mode = "memory"
optimize_mode = "memory"
##################################
# Options controlling the domain #
......
......@@ -660,13 +660,12 @@ def run_clustering_and_make_plot(
{
"general": dict(
clustering_method=method,
custom_metrics_optimize_mode=config.general.custom_metrics_optimize_mode,
custom_metrics_method=config.general.custom_metrics_method,
dtgs=dict(
list=config.general.dtgs,
cycle_length=config.general.dtgs.cycle_length.freqstr,
),
),
"metrics": config.metrics,
"clustering_method.%s"
% (method): dict(
eps=eps,
......
......@@ -379,19 +379,6 @@ with config_section("general") as section:
config_metadata.register("dtgs.start")
config_metadata.register("dtgs.end")
config_metadata.register("dtgs.cycle_length", default="3H")
# TODO: Group metrics-related options in their own config section
# Metrics optimisaion scheme
config_metadata.register(
"custom_metrics_optimize_mode",
default="memory",
choices=["speed", "memory", "speed_mem_compromise"],
)
# Metrics calculation method
config_metadata.register(
"custom_metrics_method",
default="haversine",
choices=["correlation", "haversine"],
)
# Data cols to ignore when running clustering
config_metadata.register(
"unclusterable_data_columns", default=["id", "time_utc"]
......@@ -402,7 +389,20 @@ with config_section("general") as section:
default=["pressure", "temperature", "humidity", "sum_rain_1"],
)
# "commans section"
# "metrics" section
with config_section("metrics") as section:
config_metadata.register(
"method",
default="haversine",
choices=["correlation", "haversine"],
)
config_metadata.register(
"optimize_mode",
default="memory",
choices=["speed", "memory", "speed_mem_compromise"],
)
# "commands" section
with config_section("commands.select") as section:
config_metadata.register("station_rejection_tol", default=0.15, minval=0.0)
......
......@@ -331,7 +331,7 @@ def calc_distance_matrix(df, config, domain=None, num_threads=-1):
"""
accepted_methods = ["correlation", "haversine"]
method = config.general.custom_metrics_method.lower()
method = config.metrics.method.lower()
if method not in accepted_methods:
raise NotImplementedError(
"Distance matrix calc method '%s' not available. " % (method)
......@@ -346,7 +346,7 @@ def calc_distance_matrix(df, config, domain=None, num_threads=-1):
# Drop columns that won't be used in the clustering
df=df,
weights=weights_dict_to_np_array(df, config=config),
optimize_mode=config.general.custom_metrics_optimize_mode,
optimize_mode=config.metrics.optimize_mode,
num_threads=num_threads,
)
......@@ -360,7 +360,7 @@ def calc_distance_matrix(df, config, domain=None, num_threads=-1):
return calc_distance_matrix_considering_correlation(
df=df,
weights_dict=weights_dict,
optimize_mode=config.general.custom_metrics_optimize_mode,
optimize_mode=config.metrics.optimize_mode,
num_threads=num_threads,
domain=domain,
)
......@@ -40,10 +40,10 @@ class TestsCustomMetrics:
# Prepare a mock config
fpath = Path(__file__).resolve().parent / "test_config.toml"
config = ParsedConfig(fpath)
config.general.custom_metrics_method = "haversine"
config.metrics.method = "haversine"
config.general.clustering_method = "hdbscan"
config.clustering_method.hdbscan.obs_weights = {"some_obs": 0.0}
config.general.custom_metrics_optimize_mode = "speed"
config.metrics.optimize_mode = "speed"
config.general.unclusterable_data_columns = []
logger.info("Calculating %dx%d distance matrix:", n_pts, n_pts)
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment