Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
Menu
Open sidebar
iObs
WP2
Task 2.3
netatmoqc
Commits
38a2a42f
Commit
38a2a42f
authored
Jun 04, 2021
by
Paulo Medeiros
Browse files
More linting fixes
parent
2efc8e58
Changes
10
Hide whitespace changes
Inline
Side-by-side
netatmoqc/apps/clustering/app.py
View file @
38a2a42f
...
...
@@ -210,7 +210,9 @@ def generate_control_card():
value
=
5
,
step
=
1
,
required
=
True
,
style
=
dict
(
display
=
"table-cell"
,),
style
=
dict
(
display
=
"table-cell"
,
),
),
],
style
=
dict
(
display
=
"table-cell"
),
...
...
@@ -233,7 +235,9 @@ def generate_control_card():
min
=
2
,
value
=
5
,
step
=
1
,
style
=
dict
(
display
=
"table-cell"
,),
style
=
dict
(
display
=
"table-cell"
,
),
),
],
style
=
dict
(
display
=
"table-cell"
),
...
...
@@ -250,7 +254,9 @@ def generate_control_card():
inputMode
=
"numeric"
,
min
=
0.0
,
value
=
10.0
,
style
=
dict
(
display
=
"table-cell"
,),
style
=
dict
(
display
=
"table-cell"
,
),
),
],
style
=
dict
(
display
=
"table-cell"
),
...
...
@@ -391,8 +397,14 @@ def indicator(text, id_value):
children
=
[
html
.
Div
(
children
=
[
html
.
P
(
id
=
id_value
,
className
=
"indicator_value"
,),
html
.
P
(
text
,
className
=
"indicator_text"
,),
html
.
P
(
id
=
id_value
,
className
=
"indicator_value"
,
),
html
.
P
(
text
,
className
=
"indicator_text"
,
),
],
),
],
...
...
@@ -423,7 +435,11 @@ def generate_right_column_elements():
children
=
[
html
.
Div
(
id
=
"clustering_plot_div"
,
children
=
[
dcc
.
Graph
(
id
=
"clustering_plot"
,)],
children
=
[
dcc
.
Graph
(
id
=
"clustering_plot"
,
)
],
),
html
.
Div
(
id
=
"clustered_data_table_card"
,
...
...
@@ -438,7 +454,9 @@ def generate_right_column_elements():
sort_action
=
"native"
,
filter_action
=
"native"
,
# Styling
style_cell
=
dict
(
padding
=
"10px"
,),
style_cell
=
dict
(
padding
=
"10px"
,
),
style_header
=
dict
(
backgroundColor
=
"rgb(2,21,70)"
,
color
=
"white"
,
...
...
@@ -486,7 +504,10 @@ app.layout = html.Div(
children
=
[
generate_indicators
(),
html
.
B
(
"Visualisation of Clusters"
),
html
.
Div
(
id
=
"calculated_dist"
,
children
=
[],),
html
.
Div
(
id
=
"calculated_dist"
,
children
=
[],
),
html
.
Hr
(),
dcc
.
Loading
(
# Embed the the right-hand side column inside a dcc.Loading
...
...
@@ -674,7 +695,9 @@ def run_clustering_and_make_plot(
time_start_clustering
=
time
.
time
()
logger
.
info
(
"Running %s..."
,
method
)
df
=
cluster_netatmo_obs
(
df
=
df
,
config
=
clustering_config
,
calc_silhouette_samples
=
True
,
df
=
df
,
config
=
clustering_config
,
calc_silhouette_samples
=
True
,
)
df
=
sort_df_by_cluster_size
(
df
)
time_end_clustering
=
time
.
time
()
...
...
netatmoqc/apps/scattergeo_timeseries/app.py
View file @
38a2a42f
...
...
@@ -47,7 +47,10 @@ def description_card():
children
=
[
html
.
H5
(
"NetAtmo Data Explorer"
),
html
.
H3
(
"NetAtmo Data Explorer Dashboard"
),
html
.
Div
(
id
=
"intro"
,
children
=
"An aid to explore NetAtmo data"
,),
html
.
Div
(
id
=
"intro"
,
children
=
"An aid to explore NetAtmo data"
,
),
],
style
=
{
"text-align"
:
"center"
},
)
...
...
netatmoqc/clustering.py
View file @
38a2a42f
...
...
@@ -307,7 +307,9 @@ def run_clustering_on_df(
metric
=
"precomputed"
,
).
fit
(
distance_matrix
)
logger
.
debug
(
" * Done with %s. Elapsed: %.2fs"
,
method
,
time
.
time
()
-
tstart
,
" * Done with %s. Elapsed: %.2fs"
,
method
,
time
.
time
()
-
tstart
,
)
# Update df with cluster label info. It is important that this is done
# right before calling filter_outliers, as the filter_outliers function
...
...
netatmoqc/config_parser.py
View file @
38a2a42f
...
...
@@ -357,7 +357,9 @@ def parsed_path(path):
with
config_section
(
"general"
)
as
section
:
# in/our dirs
config_metadata
.
register
(
"data_rootdir"
,
default
=
"."
,
astype
=
parsed_path
,
"data_rootdir"
,
default
=
"."
,
astype
=
parsed_path
,
)
config_metadata
.
register
(
"outdir"
,
...
...
netatmoqc/domains.py
View file @
38a2a42f
...
...
@@ -420,7 +420,10 @@ class Domain:
)
# (d) Set _grid attr
return
DomainGrid
(
xaxis
=
grid_xaxis
,
yaxis
=
grid_yaxis
,
proj
=
proj
,
tstep
=
tstep
,
xaxis
=
grid_xaxis
,
yaxis
=
grid_yaxis
,
proj
=
proj
,
tstep
=
tstep
,
)
self
.
_grid
=
init_grid
(
ngrid_lonlat
,
grid_spacing
,
ezone_ngrid
)
...
...
netatmoqc/hollow_symmetric_matrix.py
View file @
38a2a42f
...
...
@@ -588,7 +588,9 @@ class HollowSymmetricMatrix(np.lib.mixins.NDArrayOperatorsMixin):
new_data
=
self
[
indices
[:,
np
.
newaxis
],
indices
]
return
self
.
__class__
(
new_data
,
dtype
=
self
.
dtype
,
optimize_mode
=
self
.
optimize_mode
,
new_data
,
dtype
=
self
.
dtype
,
optimize_mode
=
self
.
optimize_mode
,
)
def
convert_to_dense_storage
(
self
):
...
...
netatmoqc/load_data.py
View file @
38a2a42f
...
...
@@ -354,7 +354,11 @@ def rm_overlapping_stations(df):
overlapping_stations
=
(
df
[[
"id"
,
"lat"
,
"lon"
]]
.
round
(
6
)
.
groupby
([
"lat"
,
"lon"
],
as_index
=
False
,
sort
=
False
,)
.
groupby
(
[
"lat"
,
"lon"
],
as_index
=
False
,
sort
=
False
,
)
.
filter
(
lambda
grp
:
len
(
grp
[
"id"
].
unique
())
!=
1
)[
"id"
]
.
unique
()
)
...
...
@@ -397,7 +401,11 @@ def remove_duplicates_within_cycle(df, dtg):
msg
+=
"Keeping only the one closest to the DTG: "
msg
+=
"%s obs now became %s"
logger
.
debug
(
msg
,
n_stations_with_duplicates
,
dtg
,
orig_nobs
,
new_nobs
,
msg
,
n_stations_with_duplicates
,
dtg
,
orig_nobs
,
new_nobs
,
)
return
df
...
...
netatmoqc/outlier_removal.py
View file @
38a2a42f
...
...
@@ -182,7 +182,10 @@ def _filter_outliers_iterative(
# We use "-2" as a "removed by refining methods" flag
n_removed_old
=
np
.
count_nonzero
(
df
[:,
-
1
]
==
-
2
)
df
=
_filter_outliers_iterative_one_iter
(
df
,
max_n_stdev_around_mean
,
truncate
=
trunc_perc
,
weights
=
weights
,
df
,
max_n_stdev_around_mean
,
truncate
=
trunc_perc
,
weights
=
weights
,
)
n_removed_new
=
np
.
count_nonzero
(
df
[:,
-
1
]
==
-
2
)
n_removed_this_iter
=
n_removed_new
-
n_removed_old
...
...
netatmoqc/plots.py
View file @
38a2a42f
...
...
@@ -238,8 +238,16 @@ def get_domain_fig(
traceorder
=
"reversed"
,
),
geo
=
dict
(
lataxis
=
dict
(
range
=
latrange
,
showgrid
=
True
,
dtick
=
10
,),
lonaxis
=
dict
(
range
=
lonrange
,
showgrid
=
True
,
dtick
=
15
,),
lataxis
=
dict
(
range
=
latrange
,
showgrid
=
True
,
dtick
=
10
,
),
lonaxis
=
dict
(
range
=
lonrange
,
showgrid
=
True
,
dtick
=
15
,
),
),
)
...
...
@@ -436,7 +444,10 @@ def make_clustering_fig(df, domain, **kwargs):
trace_visible
=
True
trace
=
get_obs_scattergeo_trace
(
cluster_df
,
trace_name
=
label
,
marker
=
marker
,
visible
=
trace_visible
,
cluster_df
,
trace_name
=
label
,
marker
=
marker
,
visible
=
trace_visible
,
)
fig
.
add_trace
(
trace
)
...
...
@@ -526,7 +537,9 @@ def generate_single_frame(df, dataset_var, frame_duration, frame=None):
opacity
=
0.5
,
line
=
dict
(
color
=
"black"
,
width
=
0.25
),
colorbar
=
dict
(
titleside
=
"right"
,
ticks
=
"outside"
,
showticksuffix
=
"last"
,
titleside
=
"right"
,
ticks
=
"outside"
,
showticksuffix
=
"last"
,
),
)
trace
=
get_obs_scattergeo_trace
(
df
,
marker
=
marker
)
...
...
@@ -581,7 +594,10 @@ def init_fig_dict(domain, dataset_var, frame_duration):
args
=
[
None
,
dict
(
frame
=
dict
(
duration
=
frame_duration
,
redraw
=
True
,),
frame
=
dict
(
duration
=
frame_duration
,
redraw
=
True
,
),
fromcurrent
=
True
,
transition
=
dict
(
duration
=
frame_duration
/
2
,
...
...
@@ -615,9 +631,15 @@ def init_fig_dict(domain, dataset_var, frame_duration):
y
=
0
,
pad
=
dict
(
b
=
10
,
t
=
50
),
currentvalue
=
dict
(
font
=
dict
(
size
=
20
),
prefix
=
"DTG: "
,
visible
=
True
,
xanchor
=
"right"
,
font
=
dict
(
size
=
20
),
prefix
=
"DTG: "
,
visible
=
True
,
xanchor
=
"right"
,
),
transition
=
dict
(
duration
=
frame_duration
/
2
,
easing
=
"cubic-in-out"
,
),
transition
=
dict
(
duration
=
frame_duration
/
2
,
easing
=
"cubic-in-out"
,),
steps
=
[],
)
...
...
netatmoqc/save_data.py
View file @
38a2a42f
...
...
@@ -284,7 +284,10 @@ def _input2output_single_dtg(
logger
=
get_logger
(
__name__
,
loglevel
)
logger
.
debug
(
"Reading data for %sDTG=%s%s..."
,
logcolor
.
cyan
,
dtg
,
logcolor
.
reset
,
"Reading data for %sDTG=%s%s..."
,
logcolor
.
cyan
,
dtg
,
logcolor
.
reset
,
)
try
:
# read_netatmo_data_for_dtg will raise DataNotFoundError if
...
...
@@ -385,7 +388,9 @@ def netatmoqc_input2output(
outdir
=
Path
()
logger
.
info
(
"%sSaving selected observations...%s"
,
logcolor
.
cyan
,
logcolor
.
reset
,
"%sSaving selected observations...%s"
,
logcolor
.
cyan
,
logcolor
.
reset
,
)
outdir_csv
=
None
...
...
@@ -393,7 +398,10 @@ def netatmoqc_input2output(
if
save_csv
:
outdir_csv
=
Path
(
outdir
)
/
"csv_files"
logger
.
info
(
"%s> CSV outdir:%s %s"
,
logcolor
.
cyan
,
logcolor
.
reset
,
outdir_csv
,
"%s> CSV outdir:%s %s"
,
logcolor
.
cyan
,
logcolor
.
reset
,
outdir_csv
,
)
if
save_obsoul
:
outdir_obsoul
=
Path
(
outdir
)
/
"obsoul_files"
...
...
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment