Commit c615173b authored by Paulo Medeiros's avatar Paulo Medeiros
Browse files

Minor fixes

parents 7556e72d a667f561
Pipeline #9739 passed with stages
in 3 minutes and 15 seconds
...@@ -16,16 +16,16 @@ stages: ...@@ -16,16 +16,16 @@ stages:
- ls poetry.lock || poetry lock - ls poetry.lock || poetry lock
- poetry install -v --remove-untracked - poetry install -v --remove-untracked
create_py36: create_py38:
stage: "Prepare Environments" stage: "Prepare Environments"
script: script:
- conda activate gitlab_runner_py36 2>/dev/null || conda create -y --name gitlab_runner_py36 python=3.6 - conda activate gitlab_runner_py38 2>/dev/null || conda create -y --name gitlab_runner_py38 python=3.8
- !reference [.install_deps, script] - !reference [.install_deps, script]
create_py38: create_py39:
stage: "Prepare Environments" stage: "Prepare Environments"
script: script:
- conda activate gitlab_runner_py38 2>/dev/null || conda create -y --name gitlab_runner_py38 python=3.8 - conda activate gitlab_runner_py39 2>/dev/null || conda create -y --name gitlab_runner_py39 python=3.9
- !reference [.install_deps, script] - !reference [.install_deps, script]
...@@ -42,24 +42,27 @@ create_py38: ...@@ -42,24 +42,27 @@ create_py38:
isort: isort:
stage: lint stage: lint
needs: ["create_py38"]
script: script:
- conda activate gitlab_runner_py36 - conda activate gitlab_runner_py38
- isort --check-only . - isort --check-only .
extends: extends:
- .run_lint_conditions - .run_lint_conditions
black: black:
stage: lint stage: lint
needs: ["create_py38"]
script: script:
- conda activate gitlab_runner_py36 - conda activate gitlab_runner_py38
- black --check . - black --check .
extends: extends:
- .run_lint_conditions - .run_lint_conditions
pylint: pylint:
stage: lint stage: lint
needs: ["create_py38"]
script: script:
- conda activate gitlab_runner_py36 - conda activate gitlab_runner_py38
- pylint netatmoqc - pylint netatmoqc
allow_failure: true allow_failure: true
extends: extends:
...@@ -67,8 +70,9 @@ pylint: ...@@ -67,8 +70,9 @@ pylint:
flake8: flake8:
stage: lint stage: lint
needs: ["create_py38"]
script: script:
- conda activate gitlab_runner_py36 - conda activate gitlab_runner_py38
# Set FLAKEHELL_CACHE to a random new dir. Cached data was causing issues with pylint configs. # Set FLAKEHELL_CACHE to a random new dir. Cached data was causing issues with pylint configs.
- export FLAKEHELL_CACHE="${TMPDIR:-${TEMP:-${TMP:-/tmp}}}/flakehell_cache.$(openssl rand -hex 12)" - export FLAKEHELL_CACHE="${TMPDIR:-${TEMP:-${TMP:-/tmp}}}/flakehell_cache.$(openssl rand -hex 12)"
- flake8helled . - flake8helled .
...@@ -87,16 +91,18 @@ flake8: ...@@ -87,16 +91,18 @@ flake8:
- pyproject.toml - pyproject.toml
- .gitlab-ci.yml - .gitlab-ci.yml
Unit Tests Python 3.6: Unit Tests Python 3.8:
stage: test stage: test
needs: ["create_py38"]
script: script:
- conda activate gitlab_runner_py36 - conda activate gitlab_runner_py38
- pytest -v tests - pytest -v tests
extends: .run_unit_tests_conditions extends: .run_unit_tests_conditions
Unit Tests Python 3.8: Unit Tests Python 3.9:
stage: test stage: test
needs: ["create_py39"]
script: script:
- conda activate gitlab_runner_py38 - conda activate gitlab_runner_py39
- pytest -v tests - pytest -v tests
extends: .run_unit_tests_conditions extends: .run_unit_tests_conditions
...@@ -4,7 +4,7 @@ repos: ...@@ -4,7 +4,7 @@ repos:
hooks: hooks:
- id: isort - id: isort
- repo: https://github.com/psf/black - repo: https://github.com/psf/black
rev: stable rev: 5d33f20a2a2c85cfb521ae9c5f9254bfe9fc2fd9
hooks: hooks:
- id: black - id: black
language_version: python3.6 language_version: python3.6
...@@ -15,7 +15,7 @@ repos: ...@@ -15,7 +15,7 @@ repos:
entry: pylint entry: pylint
language: system language: system
types: [python] types: [python]
- repo: https://github.com/life4/flakehell - repo: https://github.com/flakehell/flakehell
rev: fb5b9b4e744af29f85658466b17f31c7048f16b2 rev: 91a470fa2adfd0010162ee0c2b54f046c053fef7
hooks: hooks:
- id: flakehell - id: flakehell
...@@ -210,7 +210,9 @@ def generate_control_card(): ...@@ -210,7 +210,9 @@ def generate_control_card():
value=5, value=5,
step=1, step=1,
required=True, required=True,
style=dict(display="table-cell",), style=dict(
display="table-cell",
),
), ),
], ],
style=dict(display="table-cell"), style=dict(display="table-cell"),
...@@ -233,7 +235,9 @@ def generate_control_card(): ...@@ -233,7 +235,9 @@ def generate_control_card():
min=2, min=2,
value=5, value=5,
step=1, step=1,
style=dict(display="table-cell",), style=dict(
display="table-cell",
),
), ),
], ],
style=dict(display="table-cell"), style=dict(display="table-cell"),
...@@ -250,7 +254,9 @@ def generate_control_card(): ...@@ -250,7 +254,9 @@ def generate_control_card():
inputMode="numeric", inputMode="numeric",
min=0.0, min=0.0,
value=10.0, value=10.0,
style=dict(display="table-cell",), style=dict(
display="table-cell",
),
), ),
], ],
style=dict(display="table-cell"), style=dict(display="table-cell"),
...@@ -391,8 +397,14 @@ def indicator(text, id_value): ...@@ -391,8 +397,14 @@ def indicator(text, id_value):
children=[ children=[
html.Div( html.Div(
children=[ children=[
html.P(id=id_value, className="indicator_value",), html.P(
html.P(text, className="indicator_text",), id=id_value,
className="indicator_value",
),
html.P(
text,
className="indicator_text",
),
], ],
), ),
], ],
...@@ -423,7 +435,11 @@ def generate_right_column_elements(): ...@@ -423,7 +435,11 @@ def generate_right_column_elements():
children = [ children = [
html.Div( html.Div(
id="clustering_plot_div", id="clustering_plot_div",
children=[dcc.Graph(id="clustering_plot",)], children=[
dcc.Graph(
id="clustering_plot",
)
],
), ),
html.Div( html.Div(
id="clustered_data_table_card", id="clustered_data_table_card",
...@@ -438,7 +454,9 @@ def generate_right_column_elements(): ...@@ -438,7 +454,9 @@ def generate_right_column_elements():
sort_action="native", sort_action="native",
filter_action="native", filter_action="native",
# Styling # Styling
style_cell=dict(padding="10px",), style_cell=dict(
padding="10px",
),
style_header=dict( style_header=dict(
backgroundColor="rgb(2,21,70)", backgroundColor="rgb(2,21,70)",
color="white", color="white",
...@@ -486,7 +504,10 @@ app.layout = html.Div( ...@@ -486,7 +504,10 @@ app.layout = html.Div(
children=[ children=[
generate_indicators(), generate_indicators(),
html.B("Visualisation of Clusters"), html.B("Visualisation of Clusters"),
html.Div(id="calculated_dist", children=[],), html.Div(
id="calculated_dist",
children=[],
),
html.Hr(), html.Hr(),
dcc.Loading( dcc.Loading(
# Embed the the right-hand side column inside a dcc.Loading # Embed the the right-hand side column inside a dcc.Loading
...@@ -674,7 +695,9 @@ def run_clustering_and_make_plot( ...@@ -674,7 +695,9 @@ def run_clustering_and_make_plot(
time_start_clustering = time.time() time_start_clustering = time.time()
logger.info("Running %s...", method) logger.info("Running %s...", method)
df = cluster_netatmo_obs( df = cluster_netatmo_obs(
df=df, config=clustering_config, calc_silhouette_samples=True, df=df,
config=clustering_config,
calc_silhouette_samples=True,
) )
df = sort_df_by_cluster_size(df) df = sort_df_by_cluster_size(df)
time_end_clustering = time.time() time_end_clustering = time.time()
......
...@@ -47,7 +47,10 @@ def description_card(): ...@@ -47,7 +47,10 @@ def description_card():
children=[ children=[
html.H5("NetAtmo Data Explorer"), html.H5("NetAtmo Data Explorer"),
html.H3("NetAtmo Data Explorer Dashboard"), html.H3("NetAtmo Data Explorer Dashboard"),
html.Div(id="intro", children="An aid to explore NetAtmo data",), html.Div(
id="intro",
children="An aid to explore NetAtmo data",
),
], ],
style={"text-align": "center"}, style={"text-align": "center"},
) )
......
...@@ -307,7 +307,9 @@ def run_clustering_on_df( ...@@ -307,7 +307,9 @@ def run_clustering_on_df(
metric="precomputed", metric="precomputed",
).fit(distance_matrix) ).fit(distance_matrix)
logger.debug( logger.debug(
" * Done with %s. Elapsed: %.2fs", method, time.time() - tstart, " * Done with %s. Elapsed: %.2fs",
method,
time.time() - tstart,
) )
# Update df with cluster label info. It is important that this is done # Update df with cluster label info. It is important that this is done
# right before calling filter_outliers, as the filter_outliers function # right before calling filter_outliers, as the filter_outliers function
......
...@@ -145,7 +145,10 @@ def _select_stations_single_dtg(dtg, config, args): ...@@ -145,7 +145,10 @@ def _select_stations_single_dtg(dtg, config, args):
df, df_moving = remove_irregular_stations(df) df, df_moving = remove_irregular_stations(df)
df = cluster_netatmo_obs( df = cluster_netatmo_obs(
df=df, config=config, n_jobs=cpu_share, calc_silhouette_samples=False, df=df,
config=config,
n_jobs=cpu_share,
calc_silhouette_samples=False,
) )
selected_cols = ["id", "lat", "lon", "alt"] selected_cols = ["id", "lat", "lon", "alt"]
...@@ -446,6 +449,9 @@ def csv2obsoul(args): ...@@ -446,6 +449,9 @@ def csv2obsoul(args):
Args: Args:
args (argparse.Namespace): Parsed command line arguments. args (argparse.Namespace): Parsed command line arguments.
Raises:
NotImplementedError: If args.selected_stations_fpath is not a CSV.
""" """
config = read_config(args.config_file) config = read_config(args.config_file)
...@@ -581,7 +587,11 @@ def show(args): ...@@ -581,7 +587,11 @@ def show(args):
fig = domain.get_fig(display_grid_max_gsize=20000.0) fig = domain.get_fig(display_grid_max_gsize=20000.0)
fig.update_layout( fig.update_layout(
legend=dict( legend=dict(
orientation="h", xanchor="center", x=0.5, yanchor="top", y=0.0, orientation="h",
xanchor="center",
x=0.5,
yanchor="top",
y=0.0,
) )
) )
fig.show(config=DEF_FIGSHOW_CONFIG) fig.show(config=DEF_FIGSHOW_CONFIG)
......
...@@ -357,7 +357,9 @@ def parsed_path(path): ...@@ -357,7 +357,9 @@ def parsed_path(path):
with config_section("general") as section: with config_section("general") as section:
# in/our dirs # in/our dirs
config_metadata.register( config_metadata.register(
"data_rootdir", default=".", astype=parsed_path, "data_rootdir",
default=".",
astype=parsed_path,
) )
config_metadata.register( config_metadata.register(
"outdir", "outdir",
......
...@@ -420,7 +420,10 @@ class Domain: ...@@ -420,7 +420,10 @@ class Domain:
) )
# (d) Set _grid attr # (d) Set _grid attr
return DomainGrid( return DomainGrid(
xaxis=grid_xaxis, yaxis=grid_yaxis, proj=proj, tstep=tstep, xaxis=grid_xaxis,
yaxis=grid_yaxis,
proj=proj,
tstep=tstep,
) )
self._grid = init_grid(ngrid_lonlat, grid_spacing, ezone_ngrid) self._grid = init_grid(ngrid_lonlat, grid_spacing, ezone_ngrid)
......
...@@ -588,7 +588,9 @@ class HollowSymmetricMatrix(np.lib.mixins.NDArrayOperatorsMixin): ...@@ -588,7 +588,9 @@ class HollowSymmetricMatrix(np.lib.mixins.NDArrayOperatorsMixin):
new_data = self[indices[:, np.newaxis], indices] new_data = self[indices[:, np.newaxis], indices]
return self.__class__( return self.__class__(
new_data, dtype=self.dtype, optimize_mode=self.optimize_mode, new_data,
dtype=self.dtype,
optimize_mode=self.optimize_mode,
) )
def convert_to_dense_storage(self): def convert_to_dense_storage(self):
......
...@@ -354,7 +354,11 @@ def rm_overlapping_stations(df): ...@@ -354,7 +354,11 @@ def rm_overlapping_stations(df):
overlapping_stations = ( overlapping_stations = (
df[["id", "lat", "lon"]] df[["id", "lat", "lon"]]
.round(6) .round(6)
.groupby(["lat", "lon"], as_index=False, sort=False,) .groupby(
["lat", "lon"],
as_index=False,
sort=False,
)
.filter(lambda grp: len(grp["id"].unique()) != 1)["id"] .filter(lambda grp: len(grp["id"].unique()) != 1)["id"]
.unique() .unique()
) )
...@@ -397,7 +401,11 @@ def remove_duplicates_within_cycle(df, dtg): ...@@ -397,7 +401,11 @@ def remove_duplicates_within_cycle(df, dtg):
msg += "Keeping only the one closest to the DTG: " msg += "Keeping only the one closest to the DTG: "
msg += "%s obs now became %s" msg += "%s obs now became %s"
logger.debug( logger.debug(
msg, n_stations_with_duplicates, dtg, orig_nobs, new_nobs, msg,
n_stations_with_duplicates,
dtg,
orig_nobs,
new_nobs,
) )
return df return df
......
...@@ -182,7 +182,10 @@ def _filter_outliers_iterative( ...@@ -182,7 +182,10 @@ def _filter_outliers_iterative(
# We use "-2" as a "removed by refining methods" flag # We use "-2" as a "removed by refining methods" flag
n_removed_old = np.count_nonzero(df[:, -1] == -2) n_removed_old = np.count_nonzero(df[:, -1] == -2)
df = _filter_outliers_iterative_one_iter( df = _filter_outliers_iterative_one_iter(
df, max_n_stdev_around_mean, truncate=trunc_perc, weights=weights, df,
max_n_stdev_around_mean,
truncate=trunc_perc,
weights=weights,
) )
n_removed_new = np.count_nonzero(df[:, -1] == -2) n_removed_new = np.count_nonzero(df[:, -1] == -2)
n_removed_this_iter = n_removed_new - n_removed_old n_removed_this_iter = n_removed_new - n_removed_old
......
...@@ -238,8 +238,16 @@ def get_domain_fig( ...@@ -238,8 +238,16 @@ def get_domain_fig(
traceorder="reversed", traceorder="reversed",
), ),
geo=dict( geo=dict(
lataxis=dict(range=latrange, showgrid=True, dtick=10,), lataxis=dict(
lonaxis=dict(range=lonrange, showgrid=True, dtick=15,), range=latrange,
showgrid=True,
dtick=10,
),
lonaxis=dict(
range=lonrange,
showgrid=True,
dtick=15,
),
), ),
) )
...@@ -436,7 +444,10 @@ def make_clustering_fig(df, domain, **kwargs): ...@@ -436,7 +444,10 @@ def make_clustering_fig(df, domain, **kwargs):
trace_visible = True trace_visible = True
trace = get_obs_scattergeo_trace( trace = get_obs_scattergeo_trace(
cluster_df, trace_name=label, marker=marker, visible=trace_visible, cluster_df,
trace_name=label,
marker=marker,
visible=trace_visible,
) )
fig.add_trace(trace) fig.add_trace(trace)
...@@ -526,7 +537,9 @@ def generate_single_frame(df, dataset_var, frame_duration, frame=None): ...@@ -526,7 +537,9 @@ def generate_single_frame(df, dataset_var, frame_duration, frame=None):
opacity=0.5, opacity=0.5,
line=dict(color="black", width=0.25), line=dict(color="black", width=0.25),
colorbar=dict( colorbar=dict(
titleside="right", ticks="outside", showticksuffix="last", titleside="right",
ticks="outside",
showticksuffix="last",
), ),
) )
trace = get_obs_scattergeo_trace(df, marker=marker) trace = get_obs_scattergeo_trace(df, marker=marker)
...@@ -581,7 +594,10 @@ def init_fig_dict(domain, dataset_var, frame_duration): ...@@ -581,7 +594,10 @@ def init_fig_dict(domain, dataset_var, frame_duration):
args=[ args=[
None, None,
dict( dict(
frame=dict(duration=frame_duration, redraw=True,), frame=dict(
duration=frame_duration,
redraw=True,
),
fromcurrent=True, fromcurrent=True,
transition=dict( transition=dict(
duration=frame_duration / 2, duration=frame_duration / 2,
...@@ -615,9 +631,15 @@ def init_fig_dict(domain, dataset_var, frame_duration): ...@@ -615,9 +631,15 @@ def init_fig_dict(domain, dataset_var, frame_duration):
y=0, y=0,
pad=dict(b=10, t=50), pad=dict(b=10, t=50),
currentvalue=dict( currentvalue=dict(
font=dict(size=20), prefix="DTG: ", visible=True, xanchor="right", font=dict(size=20),
prefix="DTG: ",
visible=True,
xanchor="right",
),
transition=dict(
duration=frame_duration / 2,
easing="cubic-in-out",
), ),
transition=dict(duration=frame_duration / 2, easing="cubic-in-out",),
steps=[], steps=[],
) )
......
...@@ -284,7 +284,10 @@ def _input2output_single_dtg( ...@@ -284,7 +284,10 @@ def _input2output_single_dtg(
logger = get_logger(__name__, loglevel) logger = get_logger(__name__, loglevel)
logger.debug( logger.debug(
"Reading data for %sDTG=%s%s...", logcolor.cyan, dtg, logcolor.reset, "Reading data for %sDTG=%s%s...",
logcolor.cyan,
dtg,
logcolor.reset,
) )
try: try:
# read_netatmo_data_for_dtg will raise DataNotFoundError if # read_netatmo_data_for_dtg will raise DataNotFoundError if
...@@ -385,7 +388,9 @@ def netatmoqc_input2output( ...@@ -385,7 +388,9 @@ def netatmoqc_input2output(
outdir = Path() outdir = Path()
logger.info( logger.info(
"%sSaving selected observations...%s", logcolor.cyan, logcolor.reset, "%sSaving selected observations...%s",
logcolor.cyan,
logcolor.reset,
) )
outdir_csv = None outdir_csv = None
...@@ -393,7 +398,10 @@ def netatmoqc_input2output( ...@@ -393,7 +398,10 @@ def netatmoqc_input2output(
if save_csv: if save_csv:
outdir_csv = Path(outdir) / "csv_files" outdir_csv = Path(outdir) / "csv_files"
logger.info( logger.info(
"%s> CSV outdir:%s %s", logcolor.cyan, logcolor.reset, outdir_csv, "%s> CSV outdir:%s %s",
logcolor.cyan,
logcolor.reset,
outdir_csv,
) )
if save_obsoul: if save_obsoul:
outdir_obsoul = Path(outdir) / "obsoul_files" outdir_obsoul = Path(outdir) / "obsoul_files"
......
[tool.poetry] [tool.poetry]
name = "netatmoqc"