Skip to content
GitLab
Menu
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
Menu
Open sidebar
iObs
WP2
Task 2.3
netatmoqc
Commits
5fdf0eda
Commit
5fdf0eda
authored
Nov 08, 2021
by
Paulo Medeiros
Browse files
Change code linewidth limit from 79 to 90
parent
eaff0009
Changes
20
Hide whitespace changes
Inline
Side-by-side
netatmoqc/apps/clustering/app.py
View file @
5fdf0eda
...
...
@@ -18,17 +18,10 @@ from flask_caching import Cache
from
server
import
server
from
netatmoqc.clustering
import
cluster_netatmo_obs
,
sort_df_by_cluster_size
from
netatmoqc.config_parser
import
(
ParsedConfig
,
UndefinedConfigValue
,
read_config
,
)
from
netatmoqc.config_parser
import
ParsedConfig
,
UndefinedConfigValue
,
read_config
from
netatmoqc.domains
import
Domain
from
netatmoqc.dtgs
import
Dtg
from
netatmoqc.load_data
import
(
read_netatmo_data_for_dtg
,
remove_irregular_stations
,
)
from
netatmoqc.load_data
import
read_netatmo_data_for_dtg
,
remove_irregular_stations
from
netatmoqc.logs
import
CustomFormatter
from
netatmoqc.metrics
import
haversine_distance
from
netatmoqc.plots
import
make_clustering_fig
...
...
@@ -46,9 +39,7 @@ app = dash.Dash(
name
=
"clustering"
,
server
=
server
,
url_base_pathname
=
"/clustering/"
,
meta_tags
=
[
{
"name"
:
"viewport"
,
"content"
:
"width=device-width, initial-scale=1"
}
],
meta_tags
=
[{
"name"
:
"viewport"
,
"content"
:
"width=device-width, initial-scale=1"
}],
)
# Fix duplicate log items
...
...
@@ -106,13 +97,9 @@ def generate_obs_weights_panel():
# Get defaults from config file if defined. Use the ones
# defined in the calls to this function otherwise.
try
:
default_from_config
=
config
.
get_clustering_opt
(
"obs_weights"
,
{})[
var_name
]
default_from_config
=
config
.
get_clustering_opt
(
"obs_weights"
,
{})[
var_name
]
if
default_from_config
is
UndefinedConfigValue
:
raise
AttributeError
(
'obs_weights not defined for "{}"'
.
format
(
var_name
)
)
raise
AttributeError
(
'obs_weights not defined for "{}"'
.
format
(
var_name
))
default
=
default_from_config
logger
.
debug
(
'Using config file default "%s" for "%s" weight'
,
...
...
@@ -189,9 +176,7 @@ def generate_control_card():
html
.
P
(
"Clustering method"
),
dcc
.
Dropdown
(
id
=
"method-select"
,
options
=
[
{
"label"
:
i
,
"value"
:
i
}
for
i
in
allowed_cluster_methods
],
options
=
[{
"label"
:
i
,
"value"
:
i
}
for
i
in
allowed_cluster_methods
],
value
=
allowed_cluster_methods
[
0
],
),
html
.
Br
(),
...
...
@@ -298,9 +283,7 @@ def generate_control_card():
id
=
"outlier_rm_method_div"
,
children
=
[
html
.
Br
(),
html
.
P
(
"Post-Clustering Outlier Removal (Optional)"
),
html
.
P
(
"Post-Clustering Outlier Removal (Optional)"
),
dcc
.
Dropdown
(
id
=
"outlier_rm_method"
,
options
=
[{
"label"
:
"None"
,
"value"
:
None
}]
...
...
@@ -577,9 +560,7 @@ def read_data_df(str_date, cycle):
@
app
.
callback
(
[
Output
(
component_id
=
"eps_div"
,
component_property
=
"style"
),
Output
(
component_id
=
"min_cluster_size_div"
,
component_property
=
"style"
),
Output
(
component_id
=
"min_cluster_size_div"
,
component_property
=
"style"
),
],
[
Input
(
component_id
=
"method-select"
,
component_property
=
"value"
)],
)
...
...
@@ -676,9 +657,7 @@ def run_clustering_and_make_plot(
start_read_data
=
time
.
time
()
df
=
read_data_df
(
date
,
cycle
)
end_read_data
=
time
.
time
()
logger
.
info
(
"Done reading data. Elapsed: %.1fs"
,
end_read_data
-
start_read_data
)
logger
.
info
(
"Done reading data. Elapsed: %.1fs"
,
end_read_data
-
start_read_data
)
n_obs
=
len
(
df
.
index
)
if
n_obs
==
0
:
...
...
@@ -784,22 +763,14 @@ def run_clustering_and_make_plot(
data
,
dcc
.
Markdown
(
"**{:.2f}**"
.
format
(
silhouette_score
)),
dcc
.
Markdown
(
"**{}**"
.
format
(
n_clusters
)),
dcc
.
Markdown
(
"**{} ({:.2f}%)**"
.
format
(
n_accepted
,
100.0
*
n_accepted
/
n_obs
)),
dcc
.
Markdown
(
"**{} ({:.2f}%)**"
.
format
(
n_accepted
,
100.0
*
n_accepted
/
n_obs
)
),
dcc
.
Markdown
(
"**{} ({:.2f}%)**"
.
format
(
n_rm_clustering
,
100.0
*
n_rm_clustering
/
n_obs
)
),
dcc
.
Markdown
(
"**{} ({:.2f}%)**"
.
format
(
n_rm_refining
,
100.0
*
n_rm_refining
/
n_obs
)
"**{} ({:.2f}%)**"
.
format
(
n_rm_clustering
,
100.0
*
n_rm_clustering
/
n_obs
)
),
dcc
.
Markdown
(
"**{} ({:.2f}%)**"
.
format
(
n
oise_count
,
100.0
*
noise_count
/
n_obs
)
"**{} ({:.2f}%)**"
.
format
(
n
_rm_refining
,
100.0
*
n_rm_refining
/
n_obs
)
),
dcc
.
Markdown
(
"**{} ({:.2f}%)**"
.
format
(
noise_count
,
100.0
*
noise_count
/
n_obs
)),
)
...
...
@@ -809,9 +780,7 @@ def run_clustering_and_make_plot(
[
# Set clickmode='event+select' in the figure layout, and then
# use 'selectedData' here instead of 'clickData'
Input
(
component_id
=
"clustering_plot"
,
component_property
=
"selectedData"
),
Input
(
component_id
=
"clustering_plot"
,
component_property
=
"selectedData"
),
],
)
def
geodist_upon_pt_pair_selection
(
selected_data
):
...
...
netatmoqc/apps/scattergeo_timeseries/app.py
View file @
5fdf0eda
...
...
@@ -15,10 +15,7 @@ from server import server
from
netatmoqc.config_parser
import
read_config
from
netatmoqc.domains
import
Domain
from
netatmoqc.load_data
import
(
read_netatmo_data_for_dtg
,
remove_irregular_stations
,
)
from
netatmoqc.load_data
import
read_netatmo_data_for_dtg
,
remove_irregular_stations
from
netatmoqc.logs
import
CustomFormatter
from
netatmoqc.plots
import
generate_single_frame
,
init_fig_dict
...
...
@@ -34,9 +31,7 @@ app = dash.Dash(
name
=
"scattergeo_timeseries"
,
server
=
server
,
url_base_pathname
=
"/scattergeo_timeseries/"
,
meta_tags
=
[
{
"name"
:
"viewport"
,
"content"
:
"width=device-width, initial-scale=1"
}
],
meta_tags
=
[{
"name"
:
"viewport"
,
"content"
:
"width=device-width, initial-scale=1"
}],
)
...
...
@@ -92,9 +87,7 @@ def generate_control_card():
html
.
Br
(),
html
.
Div
(
id
=
"plot-btn-outer"
,
children
=
html
.
Button
(
id
=
"plot-btn"
,
children
=
"Make Plot"
,
n_clicks
=
0
),
children
=
html
.
Button
(
id
=
"plot-btn"
,
children
=
"Make Plot"
,
n_clicks
=
0
),
),
],
)
...
...
@@ -149,9 +142,7 @@ def prepare_animation(n_clicks, start_date, end_date, dataset_var):
if
n_clicks
==
0
:
return
domain
.
get_fig
(
max_ngrid
=
0
)
fig_dict
,
sliders_dict
=
init_fig_dict
(
domain
,
dataset_var
,
frame_duration
=
300
)
fig_dict
,
sliders_dict
=
init_fig_dict
(
domain
,
dataset_var
,
frame_duration
=
300
)
# Determine map boundaries and max/min plotted values
minval_dataset_var
=
float
(
"inf"
)
...
...
@@ -172,9 +163,7 @@ def prepare_animation(n_clicks, start_date, end_date, dataset_var):
for
idtg
,
dtg
in
enumerate
(
pd
.
date_range
(
start_date
,
end_date
,
freq
=
"3H"
)):
logger
.
info
(
"Reading data for %s"
,
dtg
)
df
=
read_netatmo_data_for_dtg
(
dtg
,
rootdir
=
config
.
general
.
data_rootdir
)
df
=
read_netatmo_data_for_dtg
(
dtg
,
rootdir
=
config
.
general
.
data_rootdir
)
df
,
_
=
remove_irregular_stations
(
df
)
logger
.
debug
(
" * Done. Now adding frame."
)
...
...
netatmoqc/argparse_wrapper.py
View file @
5fdf0eda
...
...
@@ -56,9 +56,7 @@ def get_parsed_args(program_name):
fpath
=
Path
(
os
.
getenv
(
"NETATMOQC_CONFIG_PATH"
,
"config.toml"
))
default_conf_path
=
fpath
.
resolve
(
strict
=
True
)
except
FileNotFoundError
:
default_conf_path
=
(
Path
(
os
.
getenv
(
"HOME"
))
/
".netatmoqc"
/
"config.toml"
)
default_conf_path
=
Path
(
os
.
getenv
(
"HOME"
))
/
".netatmoqc"
/
"config.toml"
parser
.
add_argument
(
"--version"
,
"-v"
,
action
=
"version"
,
version
=
"%(prog)s v"
+
__version__
)
...
...
netatmoqc/clustering.py
View file @
5fdf0eda
...
...
@@ -70,8 +70,7 @@ def sort_df_by_cluster_size(df):
# mess up the sorting performed above.
unique_labels
=
df
[
original_cluster_label_col
].
unique
()
_labels_old2new
=
{
old
:
new
for
new
,
old
in
enumerate
(
lab
for
lab
in
unique_labels
if
lab
>=
0
)
old
:
new
for
new
,
old
in
enumerate
(
lab
for
lab
in
unique_labels
if
lab
>=
0
)
}
@
np
.
vectorize
...
...
@@ -455,9 +454,7 @@ def cluster_netatmo_obs(df, config, **kwargs):
# Reset df: Only accepted obs will be passed on to the whole-domain
# clustering. Rejections will be added again to df after that.
df_rejected
=
df_rejoined_split
[
df_rejoined_split
[
"cluster_label"
]
<
0
].
copy
()
df_rejected
=
df_rejoined_split
[
df_rejoined_split
[
"cluster_label"
]
<
0
].
copy
()
cols_to_drop
=
[
c
for
c
in
df_rejected
.
columns
if
c
not
in
df
.
columns
]
df
=
df_rejoined_split
df
=
df
[
~
df
[
"id"
].
isin
(
df_rejected
[
"id"
])].
drop
(
cols_to_drop
,
axis
=
1
)
...
...
@@ -469,16 +466,12 @@ def cluster_netatmo_obs(df, config, **kwargs):
"DTG=%s: Main clustering over whole domain..."
,
df
.
metadata_dict
[
"dtg"
],
)
df
=
_cluster_netatmo_obs_one_domain
(
df
=
df
,
config
=
config
,
domain
=
domain
,
**
kwargs
)
df
=
_cluster_netatmo_obs_one_domain
(
df
=
df
,
config
=
config
,
domain
=
domain
,
**
kwargs
)
if
df_rejected
is
not
None
:
# Put back eventual obs rejected at the pre-clustering step
if
"original_cluster_label"
in
df
.
columns
:
df_rejected
[
"original_cluster_label"
]
=
df_rejected
[
"cluster_label"
].
copy
()
df_rejected
[
"original_cluster_label"
]
=
df_rejected
[
"cluster_label"
].
copy
()
df
=
pd
.
concat
([
df
,
df_rejected
],
ignore_index
=
True
)
# Now we're done.
...
...
netatmoqc/commands_functions.py
View file @
5fdf0eda
...
...
@@ -77,13 +77,9 @@ def cluster_obs_single_dtg(args):
outdir
.
mkdir
(
parents
=
True
)
try
:
df
=
read_netatmo_data_for_dtg
(
dtg
,
rootdir
=
config
.
general
.
data_rootdir
)
df
=
read_netatmo_data_for_dtg
(
dtg
,
rootdir
=
config
.
general
.
data_rootdir
)
except
DataNotFoundError
:
logger
.
warning
(
"Could not cluster obs for dtg=%s: "
,
dtg
,
exc_info
=
True
)
logger
.
warning
(
"Could not cluster obs for dtg=%s: "
,
dtg
,
exc_info
=
True
)
return
df
,
_
=
remove_irregular_stations
(
df
)
...
...
@@ -102,9 +98,7 @@ def cluster_obs_single_dtg(args):
if
args
.
show
:
fig
.
show
(
config
=
DEF_FIGSHOW_CONFIG
)
logger
.
info
(
"%sDone with 'cluster' command.%s"
,
logcolor
.
cyan
,
logcolor
.
reset
)
logger
.
info
(
"%sDone with 'cluster' command.%s"
,
logcolor
.
cyan
,
logcolor
.
reset
)
########################################
...
...
@@ -135,9 +129,7 @@ def _select_stations_single_dtg(dtg, config, args):
cpu_share
=
multiprocessing
.
cpu_count
()
//
proc_family_size
try
:
df
=
read_netatmo_data_for_dtg
(
dtg
,
rootdir
=
config
.
general
.
data_rootdir
)
df
=
read_netatmo_data_for_dtg
(
dtg
,
rootdir
=
config
.
general
.
data_rootdir
)
except
DataNotFoundError
:
logger
.
warning
(
"Could not select obs for dtg=%s: "
,
dtg
,
exc_info
=
True
)
return
pd
.
DataFrame
(),
pd
.
DataFrame
(),
pd
.
DataFrame
()
...
...
@@ -250,17 +242,12 @@ def select_stations(args):
# Using a rescued_rejected_stats list to modify the dataframes
# df_accepted/df_rejected is much faster than modifying
# the dataframes inside the loop above
rescued_rows
=
df_rejected
.
loc
[
df_rejected
[
"id"
].
isin
(
rescued_rejected_stats
),
:
]
rescued_rows
=
df_rejected
.
loc
[
df_rejected
[
"id"
].
isin
(
rescued_rejected_stats
),
:]
df_accepted
=
df_accepted
.
append
(
rescued_rows
,
ignore_index
=
True
)
df_rejected
=
df_rejected
.
drop
(
rescued_rows
.
index
)
if
len
(
rescued_rejected_stats
)
>
0
:
logger
.
info
(
(
" > Rescuing %d stations rejected in "
"less than %.1f%% of occurences"
),
(
" > Rescuing %d stations rejected in "
"less than %.1f%% of occurences"
),
len
(
rescued_rejected_stats
),
100
*
config
.
commands
.
select
.
station_rejection_tol
,
)
...
...
@@ -316,9 +303,7 @@ def select_stations(args):
coarse_grid_factor
=
config
.
domain
.
thinning_grid_coarse_factor
if
coarse_grid_factor
>
0
:
domain
=
Domain
.
construct_from_dict
(
config
.
domain
)
logger
.
info
(
"Thinning accepted obs: Keep only 1 station per support grid point"
)
logger
.
info
(
"Thinning accepted obs: Keep only 1 station per support grid point"
)
logger
.
info
(
" > Support grid spacing: %.1f m (%d x the domain's)"
,
domain
.
thinning_grid
.
x_spacing
,
...
...
@@ -345,9 +330,7 @@ def select_stations(args):
# (c) Thin data keeping only the first entry found at each (i, j).
# As we've sorted by rejection rate (lower to higher), all but the
# lowest-rejection-rate station at each grid (i, j) will be kept.
grid_trimmed_stations
=
domain
.
thinning_grid
.
thin_obs
(
df_accepted
,
method
=
"first"
)
grid_trimmed_stations
=
domain
.
thinning_grid
.
thin_obs
(
df_accepted
,
method
=
"first"
)
# (d) Finally, move to df_rejected those stations that appear in
# df_accepted but not in grid_trimmed_stations
...
...
@@ -495,9 +478,7 @@ def csv2obsoul(args):
obsoul_export_params
=
config
.
general
.
obsoul_export_params
,
)
logger
.
info
(
"%sDone with 'csv2obsoul' command.%s"
,
logcolor
.
cyan
,
logcolor
.
reset
)
logger
.
info
(
"%sDone with 'csv2obsoul' command.%s"
,
logcolor
.
cyan
,
logcolor
.
reset
)
######################################
...
...
@@ -531,9 +512,7 @@ def thin_data_from_csv_files(args):
for
fpath
in
file_list
:
if
fpath
.
suffix
!=
".csv"
:
logger
.
warning
(
"Only csv files supported. Skipping file '%s'"
,
fpath
)
logger
.
warning
(
"Only csv files supported. Skipping file '%s'"
,
fpath
)
continue
logger
.
info
(
"Parsing data from file %s"
,
fpath
)
...
...
@@ -608,9 +587,7 @@ def show(args):
logger
.
info
(
"Openning file '%s'"
,
fpath
)
_open_file_with_default_app
(
fpath
)
else
:
logger
.
warning
(
"Only html and csv files supported. Skipping file '%s'"
,
fpath
)
logger
.
warning
(
"Only html and csv files supported. Skipping file '%s'"
,
fpath
)
if
len
(
dataframes
)
>
0
:
fig
=
show_cmd_get_fig_from_dataframes
(
args
,
dataframes
,
domain
)
...
...
netatmoqc/config_parser.py
View file @
5fdf0eda
...
...
@@ -65,9 +65,7 @@ class UndefinedValueType:
if
self
.
_return_self_on_attr_error
:
return
self
raise
AttributeError
(
"'{}' object has no attribute '{}'"
.
format
(
self
.
__class__
.
__name__
,
item
)
"'{}' object has no attribute '{}'"
.
format
(
self
.
__class__
.
__name__
,
item
)
)
def
__copy__
(
self
):
...
...
@@ -378,9 +376,7 @@ with config_section("general") as section:
config_metadata
.
register
(
"dtgs.end"
)
config_metadata
.
register
(
"dtgs.cycle_length"
,
default
=
"3H"
)
# Data cols to ignore when running clustering
config_metadata
.
register
(
"unclusterable_data_columns"
,
default
=
[
"id"
,
"time_utc"
]
)
config_metadata
.
register
(
"unclusterable_data_columns"
,
default
=
[
"id"
,
"time_utc"
])
# Data cols to export when saving obsoul output
config_metadata
.
register
(
"obsoul_export_params"
,
...
...
@@ -464,12 +460,8 @@ with config_section("domain") as section:
# <https://hirlam.org/trac/browser/Harmonie/scr/Harmonie_domains.pm>
config_metadata
.
register
(
"nlon"
,
default
=
900
,
minval
=
1
,
astype
=
int
)
config_metadata
.
register
(
"nlat"
,
default
=
960
,
minval
=
1
,
astype
=
int
)
config_metadata
.
register
(
"lonc"
,
default
=
16.763011639
,
minval
=-
180
,
maxval
=
180
)
config_metadata
.
register
(
"latc"
,
default
=
63.489212956
,
minval
=-
90
,
maxval
=
90
)
config_metadata
.
register
(
"lonc"
,
default
=
16.763011639
,
minval
=-
180
,
maxval
=
180
)
config_metadata
.
register
(
"latc"
,
default
=
63.489212956
,
minval
=-
90
,
maxval
=
90
)
config_metadata
.
register
(
"lon0"
,
default
=
15.0
,
minval
=-
180
,
maxval
=
180
)
config_metadata
.
register
(
"lat0"
,
default
=
63.0
,
minval
=-
90
,
maxval
=
90
)
config_metadata
.
register
(
"gsize"
,
default
=
2500.0
,
minval
=
0
)
...
...
@@ -509,9 +501,7 @@ def _parse_dtg_entires(dtgs_config):
try
:
dtgs
=
DtgContainer
(
dtgs_config
[
"list"
],
cycle_length
=
cycle_length
)
except
ValueError
as
err
:
raise
ValueError
(
"Bad 'dtg.list' or 'dtg.cycle_length' config."
)
from
err
raise
ValueError
(
"Bad 'dtg.list' or 'dtg.cycle_length' config."
)
from
err
else
:
try
:
_
=
dtgs_config
[
"start"
]
...
...
@@ -527,9 +517,7 @@ def _parse_dtg_entires(dtgs_config):
cycle_length
=
cycle_length
,
)
except
(
ValueError
,
TypeError
)
as
err
:
raise
ValueError
(
"Bad 'dtg.start/end/cycle_length' config."
)
from
err
raise
ValueError
(
"Bad 'dtg.start/end/cycle_length' config."
)
from
err
return
dtgs
...
...
@@ -569,9 +557,7 @@ def _raw2parsed(raw, recog_configs=config_metadata, parent_keys=()):
if
metadata
is
UndefinedConfigValue
:
# Leave as is
logger
.
warning
(
"Config opt '%s' not recognised. Passed as is."
,
full_key
)
logger
.
warning
(
"Config opt '%s' not recognised. Passed as is."
,
full_key
)
parsed
[
key
]
=
user_val
continue
...
...
@@ -673,13 +659,9 @@ def _fill_defaults(raw, recog_configs=config_metadata, parent_keys=()):
user_value
=
parsed
[
key
]
except
KeyError
:
if
metadata
.
default
is
NoDefaultProvided
:
logger
.
debug
(
"Missing config opt '%s' with no default"
,
full_key
)
logger
.
debug
(
"Missing config opt '%s' with no default"
,
full_key
)
else
:
logger
.
debug
(
"Filling defaults for config opt '%s'"
,
full_key
)
logger
.
debug
(
"Filling defaults for config opt '%s'"
,
full_key
)
parsed
[
key
]
=
metadata
.
default
if
metadata
.
astype
:
parsed
[
key
]
=
metadata
.
astype
(
parsed
[
key
])
...
...
@@ -692,8 +674,7 @@ def _fill_defaults(raw, recog_configs=config_metadata, parent_keys=()):
parsed
[
key
]
=
_fill_defaults
(
user_value
,
metadata
,
all_keys
)
else
:
raise
TypeError
(
"Expected only _MetadataDict or ConfigDict, got %s"
%
(
type
(
metadata
))
"Expected only _MetadataDict or ConfigDict, got %s"
%
(
type
(
metadata
))
)
return
parsed
...
...
@@ -730,9 +711,7 @@ class ParsedConfig:
self
.
_parsed
=
_fill_defaults
(
_raw2parsed
(
self
.
_raw
))
# DTGs are treated a bit specially
self
.
_parsed
.
general
.
dtgs
=
_parse_dtg_entires
(
self
.
_parsed
.
general
.
dtgs
)
self
.
_parsed
.
general
.
dtgs
=
_parse_dtg_entires
(
self
.
_parsed
.
general
.
dtgs
)
self
.
_parsed
.
set_dynamic_flags
(
False
)
...
...
netatmoqc/domains.py
View file @
5fdf0eda
...
...
@@ -49,8 +49,7 @@ class Grid2D:
def
_validated_axis_info
(
axis
):
if
not
isinstance
(
axis
,
GridAxisConfig
):
raise
TypeError
(
"expected type 'GridAxisConfig', got '%s' instead"
%
(
type
(
axis
).
__name__
)
"expected type 'GridAxisConfig', got '%s' instead"
%
(
type
(
axis
).
__name__
)
)
return
axis
...
...
@@ -295,17 +294,13 @@ class DomainGrid(Grid2D):
def
thin_obs
(
self
,
df
,
method
=
"nearest"
):
"""Return df with only one entry per grid point."""
if
method
not
in
[
"nearest"
,
"first"
]:
raise
NotImplementedError
(
"'method' must be one of: 'nearest', 'first'"
)
raise
NotImplementedError
(
"'method' must be one of: 'nearest', 'first'"
)
if
len
(
df
.
index
)
==
0
:
return
df
# Add grid (i, j) info
icol
,
jcol
=
self
.
lonlat2grid
(
df
[
"lon"
].
to_numpy
(),
df
[
"lat"
].
to_numpy
()
)
icol
,
jcol
=
self
.
lonlat2grid
(
df
[
"lon"
].
to_numpy
(),
df
[
"lat"
].
to_numpy
())
df
[
"i"
]
=
icol
df
[
"j"
]
=
jcol
...
...
@@ -368,16 +363,12 @@ class Domain:
def
init_proj
(
ngrid_lonlat
,
proj_lon0_lat0
,
grid_spacing
):
"""Help routine to initialise domain projection."""
if
self
.
lmrt
and
abs
(
proj_lon0_lat0
[
1
])
>
0
:
logger
.
warning
(
"lat0 should be 0 if lmrt=True. Resetting lat0 to 0."
)
logger
.
warning
(
"lat0 should be 0 if lmrt=True. Resetting lat0 to 0."
)
proj_lon0_lat0
=
(
proj_lon0_lat0
[
0
],
0.0
)
y_range
=
ngrid_lonlat
[
1
]
*
grid_spacing
return
DomainProjection
(
name
=
self
.
_auto_choose_projname
(
lat0
=
proj_lon0_lat0
[
1
],
y_range
=
y_range
),
name
=
self
.
_auto_choose_projname
(
lat0
=
proj_lon0_lat0
[
1
],
y_range
=
y_range
),
lon0
=
proj_lon0_lat0
[
0
],
lat0
=
proj_lon0_lat0
[
1
],
)
...
...
@@ -390,9 +381,7 @@ class Domain:
def
init_grid
(
ngrid_lonlat
,
grid_spacing
,
ezone_ngrid
):
"""Help routine to initialise domain grid."""
# (a) Get projected coords of grid center
center_xy
=
proj
.
lonlat2xy
(
lon
=
center_lonlat
[
0
],
lat
=
center_lonlat
[
1
]
)
center_xy
=
proj
.
lonlat2xy
(
lon
=
center_lonlat
[
0
],
lat
=
center_lonlat
[
1
])
# (b) Grid x-axis
x_range
=
ngrid_lonlat
[
0
]
*
grid_spacing
xmin
=
center_xy
[
0
]
-
0.5
*
x_range
...
...
@@ -553,14 +542,9 @@ class Domain:
splits
=
[]
for
ilat
in
range
(
nsplit_lat
):
new_yc
=
(
self
.
grid
.
ymin
+
(
ilat
+
0.5
)
*
new_nlat
*
self
.
grid
.
y_spacing
)
new_yc
=
self
.
grid
.
ymin
+
(
ilat
+
0.5
)
*
new_nlat
*
self
.
grid
.
y_spacing
for
ilon
in
range
(
nsplit_lon
):
new_xc
=
(
self
.
grid
.
xmin
+
(
ilon
+
0.5
)
*
new_nlon
*
self
.
grid
.
x_spacing
)
new_xc
=
self
.
grid
.
xmin
+
(
ilon
+
0.5
)
*
new_nlon
*
self
.
grid
.
x_spacing
new_lonc
,
new_latc
=
self
.
proj
.
xy2lonlat
(
new_xc
,
new_yc
)
split
=
self
.
__class__
(
name
=
"{} Split {}/{}"
.
format
(
...
...
netatmoqc/dtgs.py
View file @
5fdf0eda
...
...
@@ -233,9 +233,7 @@ class Dtg(datetime):
)
if
not
self
.
compatible_with_cycle_length
(
cycle_length
):
raise
ValueError
(
"Dtg {} not compatible with cycle_length {}"
.
format
(
self
,
cycle_length
)
"Dtg {} not compatible with cycle_length {}"
.
format
(
self
,
cycle_length
)
)
self
.
_cycle_length
=
cycle_length
...
...
@@ -314,18 +312,14 @@ class DtgContainer:
# (i) data and (start, end) are mutually exclusive (avoid ambiguities)
if
data
is
None
:
if
(
start
is
None
)
or
(
end
is
None
):
raise
ValueError
(
"Need 'start' and 'end' and if 'data' is not passed"
)
raise
ValueError
(
"Need 'start' and 'end' and if 'data' is not passed"
)
# msg_which_input and input_vals will be used later on in the
# cycle_length validation
msg_which_input
=
"'start' and 'end'"
input_vals
=
[
start
,
end
]
else
:
if
(
start
is
not
None
)
or
(
end
is
not
None
):
raise
ValueError
(
"Pass either just 'data' or both 'start' and 'end'"
)
raise
ValueError
(
"Pass either just 'data' or both 'start' and 'end'"
)
msg_which_input
=
"the elements of 'data'"
input_vals
=
data
# (ii) Decide whether to get cycle_length from
...
...
@@ -369,9 +363,7 @@ class DtgContainer:
self
.
_start
=
None
self
.
_end
=
None
# Convert data to Dtg for consistency, and make it immutable
self
.
_data
=
tuple
(
Dtg
(
d
,
cycle_length
=
self
.
cycle_length
)
for
d
in
data
)
self
.
_data
=
tuple
(
Dtg
(
d
,
cycle_length
=
self
.
cycle_length
)
for
d
in
data
)
@
property
def
cycle_length
(
self
):
...
...
@@ -394,9 +386,7 @@ class DtgContainer:
def
calc_nth_item
(
n
):
if
(
n
>
len
(
self
)
-
1
)
or
(
n
<
-
len
(
self
)):
raise
IndexError
(
"{} index out of range"
.
format
(
self
.
__class__
.
__name__
)
)
raise
IndexError
(
"{} index out of range"
.
format
(
self
.
__class__
.
__name__
))
return
self
.
_start
+
sign
*
(
n
%
len
(
self
))
*
self
.
cycle_length
if
isinstance
(
item
,
slice
):
...
...
netatmoqc/hollow_symmetric_matrix.py
View file @
5fdf0eda
...
...
@@ -130,10 +130,7 @@ def _data_index_to_matrix_index(n, i_data, check_bounds=True):
if
i_data
<
0
or
i_data
>=
n
*
(
n
-
1
)
/
2
:
raise
ValueError
(
"Arg 'i_data' not in the range [0, n*(n-1)/2)"
)
i
=
(
int
(
0.5
*
((
2
*
n
+
1
)
-
np
.
sqrt
((
2
*
n
+
1
)
**
2
-
8
*
(
n
+
i_data
))))
-
1
)
i
=
int
(
0.5
*
((
2
*
n
+
1
)
-
np
.
sqrt
((
2
*
n
+
1
)
**
2
-
8
*
(
n
+
i_data
))))
-
1
j
=
(
1
+
i_data
+
i
)
-
(
i
*
(
2
*
n
-
i
-
1
))
//
2
return
i
,
j
...
...
@@ -417,9 +414,7 @@ class HollowSymmetricMatrix(np.lib.mixins.NDArrayOperatorsMixin):
order
=
0.5
*
(
1.0
+
np
.
sqrt
(
1.0
+
8.0
*
n_indep
))
nearest_int_order
=
int
(
np
.
rint
(
order
))
if
abs
(
order
-
nearest_int_order
)
>
1e-8
:
nearest_n_indep
=
int
(
0.5
*
nearest_int_order
*
(
nearest_int_order
-
1
)
)
nearest_n_indep
=
int
(
0.5
*
nearest_int_order
*
(
nearest_int_order
-
1
))
msg
=
"len(data)={0} incompatible with {1}: {2} elements "
msg
+=
"needed for a {3}x{3} matrix (closest option)"
msg
=
msg
.
format
(
...
...
@@ -448,8 +443,7 @@ class HollowSymmetricMatrix(np.lib.mixins.NDArrayOperatorsMixin):
np
.
fill_diagonal
(
data
,
0
)
else
:
raise
ValueError
(
"%s: ndim(data) should be 1 or 2. Got %s."
%
(
cls
.
__name__
,
np
.
ndim
(
data
))
"%s: ndim(data) should be 1 or 2. Got %s."
%
(
cls
.
__name__
,
np
.
ndim
(
data
))
)
return
data
...
...
@@ -556,9 +550,7 @@ class HollowSymmetricMatrix(np.lib.mixins.NDArrayOperatorsMixin):
if
self
.
stored_as_dense
:
rtn
=
self
.
_data