"""
QC of individual reports.
Module containing main QC functions which could be applied on a DataBundle.
"""
from __future__ import annotations
from typing import Any, Literal
import numpy as np
from .astronomical_geometry import sunangle
from .auxiliary import (
ScalarNumberType,
ValueDatetimeType,
ValueFloatType,
ValueIntType,
ValueNumberType,
convert_units,
ensure_arrays,
failed,
format_return_type,
inspect_arrays,
isvalid,
passed,
post_format_return_type,
untestable,
)
from .external_clim import ClimArgType, inspect_climatology
from .time_control import convert_date, day_in_year, get_month_lengths
vectorized_day_in_year = np.vectorize(day_in_year)
vectorized_sunangle = np.vectorize(sunangle, otypes=[float, float, float, float, float, float])
[docs]
@post_format_return_type(["value"])
@inspect_arrays(["value"])
def value_check(value: ValueNumberType) -> ValueIntType:
"""
Check if a value is equal to None or numerically invalid (NaN).
Parameters
----------
value : :py:obj:`~marine_qc.ValueNumberType`
The input value(s) to be tested.
Can be a scalar, sequence (e.g., list or tuple), a one-dimensional NumPy array, or a pandas Series.
Returns
-------
:py:obj:`~marine_qc.ValueIntType`
Same type as input, but with integer values
- Returns 1 (or array/sequence/Series of 1s) if the input value is None or numerically invalid (NaN)
- Returns 0 (or array/sequence/Series of 0s) otherwise.
Raises
------
TypeError
If `inspect_arrays` does not return np.ndarrays.
"""
(value_arr,) = ensure_arrays(value=value)
valid_mask = isvalid(value_arr)
result = np.where(valid_mask, passed, failed)
return result
[docs]
@post_format_return_type(["lat", "lon"])
@inspect_arrays(["lat", "lon"])
@convert_units(lat="degrees", lon="degrees")
def do_position_check(lat: ValueNumberType, lon: ValueNumberType) -> ValueIntType:
"""
Perform the positional QC check on the report.
Simple check to make sure that the latitude and longitude are within specified bounds:
- Latitude is between -90 and 90.
- Longitude is between 180 and 360.
Parameters
----------
lat : :py:obj:`~marine_qc.ValueNumberType`
Latitude(s) of observation in degrees.
Can be a scalar, a sequence (e.g., list or tuple), a one-dimensional NumPy array, or a pandas Series.
lon : :py:obj:`~marine_qc.ValueNumberType`
Longitude() of observation in degrees.
Can be a scalar, a sequence (e.g., list or tuple), a one-dimensional NumPy array, or a pandas Series.
Returns
-------
:py:obj:`~marine_qc.ValueIntType`
Same type as input, but with integer values
- Returns 2 (or array/sequence/Series of 2s) if either latitude or longitude is numerically invalid (None/NaN).
- Returns 1 (or array/sequence/Series of 1s) if either latitude or longitude is out of the valid range.
- Returns 0 (or array/sequence/Series of 0s) otherwise.
Raises
------
TypeError
If decorator `inspect_arrays` does not return np.ndarrays.
"""
lat_arr, lon_arr = ensure_arrays(lat=lat, lon=lon)
result = np.full(lat_arr.shape, untestable, dtype=int)
valid = isvalid(lat_arr) & isvalid(lon_arr)
lat_valid = np.where(valid, lat_arr, np.nan)
lon_valid = np.where(valid, lon_arr, np.nan)
cond_failed = (lat_valid < -90.0) | (lat_valid > 90.0) | (lon_valid < -180.0) | (lon_valid > 360.0)
result[valid & cond_failed] = failed
result[valid & ~cond_failed] = passed
return result
[docs]
@post_format_return_type(["date", "year"])
@convert_date(["year", "month", "day"])
@inspect_arrays(["year", "month", "day"])
def do_date_check(
date: ValueDatetimeType = None,
year: ValueIntType = None,
month: ValueIntType = None,
day: ValueIntType = None,
year_init: int | None = None,
year_end: int | None = None,
) -> ValueIntType:
"""
Perform the date QC check on the report. Checks whether the given date or date components are valid.
Parameters
----------
date : :py:obj:`~marine_qc.ValueDatetimeType`, optional
Date(s) of observation.
Can be a scalar, a sequence (e.g., list or tuple), a one-dimensional NumPy array, or a pandas Series.
year : :py:obj:`~marine_qc.ValueIntType`, optional
Year(s) of observation.
Can be a scalar, a sequence (e.g., list or tuple), a one-dimensional NumPy array, or a pandas Series.
month : :py:obj:`~marine_qc.ValueIntType`, optional
Month(s) of observation (1-12).
Can be a scalar, a sequence (e.g., list or tuple), a one-dimensional NumPy array, or a pandas Series.
day : :py:obj:`~marine_qc.ValueIntType`, optional
Day(s) of observation.
Can be a scalar, a sequence (e.g., list or tuple), a one-dimensional NumPy array, or a pandas Series.
year_init : int, optional
Initial valid year.
year_end : int, optional
Last valid year.
Returns
-------
:py:obj:`~marine_qc.ValueIntType`
Same type as input, but with integer values
- Returns 2 (or array/sequence/Series of 2s) if any of year, month, or day is numerically invalid or None,
- Returns 1 (or array/sequence/Series of 1s) if the date is not valid,
- Returns 0 (or array/sequence/Series of 0s) otherwise.
Raises
------
TypeError
If decorator `inspect_arrays` does not return np.ndarrays.
"""
year_arr, month_arr, day_arr = ensure_arrays(year=year, month=month, day=day)
result = np.full(year_arr.shape, untestable, dtype=int)
valid = isvalid(year_arr) & isvalid(month_arr) & isvalid(day_arr)
year_valid = year_arr[valid].astype(int)
month_valid = month_arr[valid].astype(int)
day_valid = day_arr[valid].astype(int)
result_valid = np.full(year_valid.shape, failed, dtype=int)
year_ok = np.full(year_valid.shape, True, dtype=bool)
if year_init:
year_ok[year_valid < year_init] = False
if year_end:
year_ok[year_valid > year_end] = False
month_ok = (month_valid >= 1) & (month_valid <= 12)
unique_years = np.unique(year_valid)
month_length_map = {y: get_month_lengths(y) for y in unique_years}
max_days = np.array([month_length_map[y][m - 1] for y, m in zip(year_valid, month_valid, strict=False)])
day_ok = (day_valid >= 1) & (day_valid <= max_days)
passed_mask = year_ok & month_ok & day_ok
result_valid[passed_mask] = passed
result[valid] = result_valid
return result
[docs]
@post_format_return_type(["date", "hour"])
@convert_date(["hour"])
@inspect_arrays(["hour"])
def do_time_check(
date: ValueDatetimeType = None,
hour: ValueFloatType = None,
) -> ValueIntType:
"""
Check that the time is valid i.e. in the range 0.0 to 23.99999...
Parameters
----------
date : :py:obj:`~marine_qc.ValueDatetimeType`, optional
Date(s) of observation.
Can be a scalar, a sequence (e.g., list or tuple), a one-dimensional NumPy array, or a pandas Series.
hour : :py:obj:`~marine_qc.ValueFloatType`, optional
Hour(s) of observation (minutes as decimal).
Can be a scalar, a sequence (e.g., list or tuple), a one-dimensional NumPy array, or a pandas Series.
Returns
-------
:py:obj:`~marine_qc.ValueIntType`
Same type as input, but with integer values
- Returns 2 (or array/sequence/Series of 2s) if hour is numerically invalid or None,
- Returns 1 (or array/sequence/Series of 1s) if hour is not a valid hour,
- Returns 0 (or array/sequence/Series of 0s) otherwise.
Raises
------
TypeError
If decorator `inspect_arrays` does not return np.ndarrays.
"""
(hour_arr,) = ensure_arrays(hour=hour)
result = np.full(hour_arr.shape, untestable, dtype=int)
valid_indices = isvalid(hour_arr)
cond_failed = np.full(hour_arr.shape, True, dtype=bool)
cond_failed[valid_indices] = (hour_arr[valid_indices] >= 24) | (hour_arr[valid_indices] < 0)
result[valid_indices & cond_failed] = failed
result[valid_indices & ~cond_failed] = passed
return result
[docs]
def _do_daytime_check(
year: np.ndarray,
month: np.ndarray,
day: np.ndarray,
hour: np.ndarray,
lat: np.ndarray,
lon: np.ndarray,
time_since_sun_above_horizon: ScalarNumberType,
mode: Literal["day", "night"],
) -> np.ndarray:
"""
Determine if the sun was above the horizon a specified time before the report.
Parameters
----------
year : 1D np.ndarray of int
Year(s) of observation.
month : 1D np.ndarray of int
Month(s) of observation (1-12).
day : 1D np.ndarray of int
Day(s) of observation.
hour : 1D np.ndarray of float
Hour(s) of observation (minutes as decimal).
lat : 1D np.ndarray of float
Latitude(s) of observation in degrees.
lon : 1D np.ndarray of float
Longitude() of observation in degree.
time_since_sun_above_horizon : float
Maximum time sun can have been above horizon (or below) to still count as night. Original QC test had this set
to 1.0 i.e. it was night between one hour after sundown and one hour after sunrise.
mode : {"day", "night"}
If "day", check if the sun is above the horizon.
If "night", check if the sun is below the horizon.
Returns
-------
np.ndarray of int
- Returns 2 (or array/sequence/Series of 2s) if any of do_position_check, do_date_check, or do_time_check
returns 2.
- Returns 1 (or array/sequence/Series of 1s) if any of do_position_check, do_date_check, or do_time_check
returns 1 or if it is night (sun below horizon an hour ago).
- Returns 0 if it is day (sun above horizon an hour ago).
Raises
------
ValueError
If `mode` is not in valid list ["day", "night"].
"""
if mode not in ["day", "night"]:
raise ValueError(f"mode: {mode} is not in valid list ['day', 'night']")
p_check = np.atleast_1d(do_position_check(lat, lon))
d_check = np.atleast_1d(do_date_check(year=year, month=month, day=day))
t_check = np.atleast_1d(do_time_check(hour=hour))
result = np.full(year.shape, untestable, dtype=int)
if mode == "day":
_failed = failed
_passed = passed
else:
_failed = passed
_passed = failed
failed_mask = (p_check == failed) | (d_check == failed) | (t_check == failed)
result[failed_mask] = failed
valid_mask = (~failed_mask) & (p_check != untestable) & (d_check != untestable) & (t_check != untestable)
if not np.any(valid_mask):
return result
valid_indices = np.where(valid_mask)[0]
year_valid = year[valid_indices].astype(int)
month_valid = month[valid_indices].astype(int)
day_valid = day[valid_indices].astype(int)
hour_valid = hour[valid_indices]
doy = vectorized_day_in_year(year_valid, month_valid, day_valid)
hour_whole = np.floor(hour_valid)
minute_valid = (hour_valid - hour_whole) * 60.0
if time_since_sun_above_horizon is not None:
hour_whole -= time_since_sun_above_horizon
lat_fixed = lat[valid_indices]
lat_fixed[lat_fixed == 0] = 0.0001
lon_fixed = lon[valid_indices]
lon_fixed[lon_fixed == 0] = 0.0001
underflow = hour_whole < 0
hour_whole[underflow] += 24
doy[underflow] -= 1
fix_indices = underflow & (doy <= 0)
if np.any(fix_indices):
year_valid[fix_indices] -= 1
doy[fix_indices] = vectorized_day_in_year(year_valid[fix_indices], 12, 31)
_azimuths, elevations, _rtas, _hras, _sids, _decs = vectorized_sunangle(
year_valid,
doy.astype(int),
hour_whole.astype(int),
minute_valid,
0,
0,
0,
lat_fixed,
lon_fixed,
)
result[valid_indices] = np.where(elevations > 0, _passed, _failed)
return result
[docs]
@post_format_return_type(["date", "year"])
@convert_date(["year", "month", "day", "hour"])
@inspect_arrays(["year", "month", "day", "hour", "lat", "lon"])
@convert_units(lat="degrees", lon="degrees")
def do_day_check(
date: ValueDatetimeType = None,
year: ValueIntType = None,
month: ValueIntType = None,
day: ValueIntType = None,
hour: ValueFloatType = None,
lat: ValueNumberType = None,
lon: ValueNumberType = None,
time_since_sun_above_horizon: ScalarNumberType = None,
) -> ValueIntType:
"""
Determine if the sun was above the horizon a specified time before the report.
This "day" test is used to classify Marine Air Temperature (MAT) measurements as either
Night MAT (NMAT) or Day MAT, accounting for solar heating biases and a potential lag between sun rise and the
onset of significant warming. The function calculates the sun's elevation using the `sunangle` function, offset by the
specified `time_since_sun_above_horizon`.
Parameters
----------
date : :py:obj:`~marine_qc.ValueDatetimeType`, optional
Date(s) of observation.
Can be a scalar, a sequence (e.g., list or tuple), a one-dimensional NumPy array, or a pandas Series.
year : :py:obj:`~marine_qc.ValueIntType`, optional
Year(s) of observation.
Can be a scalar, a sequence (e.g., list or tuple), a one-dimensional NumPy array, or a pandas Series.
month : :py:obj:`~marine_qc.ValueIntType`, optional
Month(s) of observation (1-12).
Can be a scalar, a sequence (e.g., list or tuple), a one-dimensional NumPy array, or a pandas Series.
day : :py:obj:`~marine_qc.ValueIntType`, optional
Day(s) of observation.
Can be a scalar, a sequence (e.g., list or tuple), a one-dimensional NumPy array, or a pandas Series.
hour : :py:obj:`~marine_qc.ValueFloatType`, optional
Hour(s) of observation (minutes as decimal).
Can be a scalar, a sequence (e.g., list or tuple), a one-dimensional NumPy array, or a pandas Series.
lat : :py:obj:`~marine_qc.ValueNumberType`, optional
Latitude(s) of observation in degrees.
Can be a scalar, a sequence (e.g., list or tuple), a one-dimensional NumPy array, or a pandas Series.
lon : :py:obj:`~marine_qc.ValueNumberType`, optional
Longitude() of observation in degree.
Can be a scalar, a sequence (e.g., list or tuple), a one-dimensional NumPy array, or a pandas Series.
time_since_sun_above_horizon : float
Maximum time sun can have been above horizon (or below) to still count as night. Original QC test had this set
to 1.0 i.e. it was night between one hour after sundown and one hour after sunrise.
Returns
-------
:py:obj:`~marine_qc.ValueIntType`
Same type as input, but with integer values
- Returns 2 (or array/sequence/Series of 2s) if any of do_position_check, do_date_check, or do_time_check
returns 2.
- Returns 1 (or array/sequence/Series of 1s) if any of do_position_check, do_date_check, or do_time_check
returns 1 or if it is night (sun below horizon an hour ago).
- Returns 0 if it is day (sun above horizon an hour ago).
Raises
------
TypeError
If decorator `inspect_arrays` does not return np.ndarrays.
See Also
--------
do_night_check: Determine if the sun was above the horizon an hour ago based on date, time, and position.
Notes
-----
In previous versions, ``time_since_sun_above_horizon`` has the default value 1.0 as one hour is used as a
definition of "day" for marine air temperature QC. Solar heating biases were considered to be negligible mmore
than one hour after sunset and up to one hour after sunrise.
"""
year_arr, month_arr, day_arr, hour_arr, lat_arr, lon_arr = ensure_arrays(year=year, month=month, day=day, hour=hour, lat=lat, lon=lon)
return _do_daytime_check(year_arr, month_arr, day_arr, hour_arr, lat_arr, lon_arr, time_since_sun_above_horizon, mode="day")
[docs]
@post_format_return_type(["date", "year"])
@convert_date(["year", "month", "day", "hour"])
@inspect_arrays(["year", "month", "day", "hour", "lat", "lon"])
@convert_units(lat="degrees", lon="degrees")
def do_night_check(
date: ValueDatetimeType = None,
year: ValueIntType = None,
month: ValueIntType = None,
day: ValueIntType = None,
hour: ValueFloatType = None,
lat: ValueNumberType = None,
lon: ValueNumberType = None,
time_since_sun_above_horizon: ScalarNumberType = None,
) -> ValueIntType:
"""
Determine if the sun was below the horizon a specified time before the report.
This "night" test is used to classify Marine Air Temperature (MAT) measurements as either
Night MAT (NMAT) or Day MAT, accounting for solar heating biases and a potential lag between sun rise and the
onset of significant warming. The function calculates the sun's elevation using the `sunangle` function, offset by the
specified `time_since_sun_above_horizon`.
Parameters
----------
date : :py:obj:`~marine_qc.ValueDatetimeType`, optional
Date(s) of observation.
Can be a scalar, a sequence (e.g., list or tuple), a one-dimensional NumPy array, or a pandas Series.
year : :py:obj:`~marine_qc.ValueIntType`, optional
Year(s) of observation.
Can be a scalar, a sequence (e.g., list or tuple), a one-dimensional NumPy array, or a pandas Series.
month : :py:obj:`~marine_qc.ValueIntType`, optional
Month(s) of observation (1-12).
Can be a scalar, a sequence (e.g., list or tuple), a one-dimensional NumPy array, or a pandas Series.
day : :py:obj:`~marine_qc.ValueIntType`, optional
Day(s) of observation.
Can be a scalar, a sequence (e.g., list or tuple), a one-dimensional NumPy array, or a pandas Series.
hour : :py:obj:`~marine_qc.ValueFloatType`, optional
Hour(s) of observation (minutes as decimal).
Can be a scalar, a sequence (e.g., list or tuple), a one-dimensional NumPy array, or a pandas Series.
lat : :py:obj:`~marine_qc.ValueNumberType`, optional
Latitude(s) of observation in degrees.
Can be a scalar, a sequence (e.g., list or tuple), a one-dimensional NumPy array, or a pandas Series.
lon : :py:obj:`~marine_qc.ValueNumberType`, optionalt
Longitude() of observation in degree.
Can be a scalar, a sequence (e.g., list or tuple), a one-dimensional NumPy array, or a pandas Series.
time_since_sun_above_horizon : float
Maximum time sun can have been above horizon (or below) to still count as night. Original QC test had this set
to 1.0 i.e. it was night between one hour after sundown and one hour after sunrise.
Returns
-------
:py:obj:`~marine_qc.ValueIntType`
Same type as input, but with integer values
- Returns 2 (or array/sequence/Series of 2s) if any of do_position_check, do_date_check, or do_time_check
returns 2.
- Returns 1 (or array/sequence/Series of 1s) if any of do_position_check, do_date_check, or do_time_check
returns 1 or if it is day (sun above horizon an hour ago).
- Returns 0 if it is night (sun below horizon an hour ago).
Raises
------
ValueError
If `mode` is not in valid list ["day", "night"].
TypeError
If decorator `inspect_arrays` does not return np.ndarrays.
See Also
--------
do_day_check: Determine if the sun was above the horizon an hour ago based on date, time, and position.
Notes
-----
In previous versions, ``time_since_sun_above_horizon`` has the default value 1.0 as one hour is used as a
definition of "day" for marine air temperature QC. Solar heating biases were considered to be negligible mmore
than one hour after sunset and up to one hour after sunrise.
"""
year_arr, month_arr, day_arr, hour_arr, lat_arr, lon_arr = ensure_arrays(year=year, month=month, day=day, hour=hour, lat=lat, lon=lon)
return _do_daytime_check(
year_arr,
month_arr,
day_arr,
hour_arr,
lat_arr,
lon_arr,
time_since_sun_above_horizon,
mode="night",
)
[docs]
def do_missing_value_check(value: ValueNumberType) -> ValueIntType:
"""
Check if a value is equal to None or numerically invalid (NaN).
Parameters
----------
value : :py:obj:`~marine_qc.ValueNumberType`
The input value(s) to be tested.
Can be a scalar, sequence (e.g., list or tuple), a one-dimensional NumPy array, or a pandas Series.
Returns
-------
:py:obj:`~marine_qc.ValueIntType`
Same type as input, but with integer values
- Returns 1 (or array/sequence/Series of 1s) if the input value is None or numerically invalid (NaN)
- Returns 0 (or array/sequence/Series of 0s) otherwise.
Raises
------
TypeError
If decorator `inspect_arrays` in :py:func:`value_check` does not return np.ndarrays.
"""
return value_check(value)
[docs]
@inspect_climatology("climatology")
def do_missing_value_clim_check(climatology: ClimArgType, **kwargs: Any) -> ValueIntType:
r"""
Check if a climatological value is equal to None or numerically invalid (NaN).
Parameters
----------
climatology : :py:obj:`~marine_qc.ClimArgType`
The input climatological value(s) to be tested.
Can be a scalar, sequence, a one-dimensional NumPy array, a pandas Series,
a :py:class:`~marine_qc.Climatology`, a path-like string on disk, a xarray Dataset or a xarray DataArray.
\**kwargs : dict
Additional keyword arguments passed by the decorator framework (unused).
Returns
-------
:py:obj:`~marine_qc.ValueIntType`
Same type as input, but with integer values
- Returns 1 (or array/sequence/Series of 1s) if the input value is None or numerically invalid (NaN)
- Returns 0 (or array/sequence/Series of 0s) otherwise.
Raises
------
TypeError
If decorator `inspect_arrays` in :py:func:`value_check` does not return np.ndarrays.
Notes
-----
If `climatology` is a :py:class:`~marine_qc.Climatology` object, pass `lon` and `lat` and `date`, or `month` and `day`,
as keyword arguments to extract the relevant climatological value.
"""
return value_check(climatology)
[docs]
@post_format_return_type(["value"])
@inspect_arrays(["value"])
@convert_units(value="unknown", limits="unknown")
def do_hard_limit_check(
value: ValueNumberType,
limits: tuple[int | float, int | float],
) -> ValueIntType:
"""
Check if a value is outside specified limits.
Parameters
----------
value : :py:obj:`~marine_qc.ValueNumberType`
The value(s) to be tested against the limits.
Can be a scalar, a sequence (e.g., list or tuple), a one-dimensional NumPy array, or a pandas Series.
limits : tuple of float
A tuple of two floats representing the lower and upper limit.
Returns
-------
:py:obj:`~marine_qc.ValueIntType`
Same type as input, but with integer values
- Returns 2 (or array/sequence/Series of 2s) if the upper limit is less than or equal
to the lower limit, or if the input is invalid (None or NaN).
- Returns 1 (or array/sequence/Series of 1s) if value(s) are outside the specified limits.
- Returns 0 (or array/sequence/Series of 0s) if value(s) are within limits.
Raises
------
TypeError
If decorator `inspect_arrays` does not return np.ndarrays.
"""
(value_arr,) = ensure_arrays(value=value)
result = np.full(value_arr.shape, untestable, dtype=int)
if limits[1] <= limits[0]:
return format_return_type(result, value_arr)
valid_indices = isvalid(value_arr)
cond_passed = np.full(value_arr.shape, True, dtype=bool)
cond_passed[valid_indices] = (limits[0] <= value_arr[valid_indices]) & (value_arr[valid_indices] <= limits[1])
result[valid_indices & cond_passed] = passed
result[valid_indices & ~cond_passed] = failed
return result
[docs]
@post_format_return_type(["value"])
@inspect_arrays(["value", "climatology"])
@convert_units(value="unknown", climatology="unknown")
@inspect_climatology("climatology", optional="standard_deviation")
def do_climatology_check(
value: ValueNumberType,
climatology: ClimArgType,
maximum_anomaly: float,
standard_deviation: ClimArgType = "default",
standard_deviation_limits: tuple[int | float, int | float] | None = None,
lowbar: int | float | None = None,
) -> ValueIntType:
"""
Climatology check to compare a value with a climatological average within specified anomaly limits.
This check supports optional parameters to customize the comparison.
If ``standard_deviation`` is provided, the value is converted into a standardised anomaly. Optionally,
if ``standard deviation`` is outside the range specified by ``standard_deviation_limits`` then
``standard_deviation`` is set to whichever of the lower or upper limits is closest.
If ``lowbar`` is provided, the anomaly must be greater than ``lowbar`` to fail regardless of ``standard_deviation``.
Parameters
----------
value : :py:obj:`~marine_qc.ValueNumberType`
Value(s) to be compared to climatology.
Can be a scalar, a sequence (e.g., list or tuple), a one-dimensional NumPy array, or a pandas Series.
climatology : :py:obj:`~marine_qc.ClimArgType`
The climatological average(s) to which the values(s) will be compared.
Can be a scalar, sequence, a one-dimensional NumPy array, a pandas Series,
a :py:class:`~marine_qc.Climatology`, a path-like string on disk, a xarray Dataset or a xarray DataArray.
maximum_anomaly : float
Largest allowed anomaly.
If ``standard_deviation`` is provided, this is interpreted as the largest allowed standardised anomaly.
standard_deviation : :py:obj:`~marine_qc.ClimArgType`, default: "default"
The standard deviation(s) used to standardise the anomaly
If set to "default", it is internally treated as 1.0.
Can be a scalar, a sequence (e.g., list or tuple), a one-dimensional NumPy array, or a pandas Series,
a :py:class:`~marine_qc.Climatology`, a path-like string on disk, a xarray Dataset or a xarray DataArray.
standard_deviation_limits : tuple of float, optional
A tuple of two floats representing the upper and lower limits for standard deviation used in the check.
lowbar : float, optional
The anomaly must be greater than lowbar to fail regardless of standard deviation.
Returns
-------
:py:obj:`~marine_qc.ValueIntType`
Same type as input, but with integer values
- Returns 2 (or array/sequence/Series of 2s) if `standard_deviation_limits[1]` is less than or equal to
`standard_deviation_limits[0]`, or if `maximum_anomaly` is less than or equal to 0, or if any of
`value`, `climate_normal`, or `standard_deviation` is numerically invalid (None or NaN).
- Returns 1 (or array/sequence/Series of 1s) if the difference is outside the specified range.
- Returns 0 (or array/sequence/Series of 0s) otherwise.
Raises
------
TypeError
If decorator `inspect_arrays` does not return np.ndarrays.
Notes
-----
If either `climatology` or `standard_deviation` is a :py:class:`~marine_qc.Climatology` object, pass `lon` and `lat`
and `date`, or `month` and `day`, as keyword arguments to extract the relevant climatological value(s).
"""
value_arr, climatology_arr = ensure_arrays(value=value, climatology=climatology)
if climatology_arr.ndim == 0:
climatology_arr = np.full_like(value_arr, climatology_arr)
if isinstance(standard_deviation, str) and standard_deviation == "default":
standard_deviation_arr: np.ndarray = np.full(value_arr.shape, 1.0, dtype=float)
else:
standard_deviation_arr = np.array(standard_deviation)
standard_deviation_arr = np.atleast_1d(standard_deviation_arr)
result = np.full(value_arr.shape, untestable, dtype=int)
if maximum_anomaly is None or maximum_anomaly <= 0:
return format_return_type(result, value_arr)
if standard_deviation_limits is None:
standard_deviation_limits = (0, np.inf)
elif standard_deviation_limits[1] <= standard_deviation_limits[0]:
return format_return_type(result, value_arr)
valid_indices = isvalid(value_arr) & isvalid(climatology_arr) & isvalid(maximum_anomaly) & isvalid(standard_deviation_arr)
standard_deviation_arr[valid_indices] = np.clip(
standard_deviation_arr[valid_indices],
standard_deviation_limits[0],
standard_deviation_limits[1],
)
climate_diff = np.zeros_like(value_arr)
climate_diff[valid_indices] = np.abs(value_arr[valid_indices] - climatology_arr[valid_indices])
if lowbar is None:
low_check = np.ones(value_arr.shape, dtype=bool)
else:
low_check = climate_diff > lowbar
cond_failed = np.full(value_arr.shape, False, dtype=bool)
cond_failed[valid_indices] = (climate_diff[valid_indices] / standard_deviation_arr[valid_indices] > maximum_anomaly) & low_check[valid_indices]
result[valid_indices & cond_failed] = failed
result[valid_indices & ~cond_failed] = passed
return result
[docs]
@post_format_return_type(["dpt", "at2"])
@inspect_arrays(["dpt", "at2"])
@convert_units(dpt="K", at2="K")
def do_supersaturation_check(dpt: ValueNumberType, at2: ValueNumberType) -> ValueIntType:
"""
Perform the super saturation check.
Check if a valid dewpoint temperature is greater than a valid air temperature.
Parameters
----------
dpt : :py:obj:`~marine_qc.ValueNumberType`
Dewpoint temperature value(s).
Can be a scalar, a sequence (e.g., list or tuple), a one-dimensional NumPy array, or a pandas Series.
at2 : :py:obj:`~marine_qc.ValueNumberType`
Air temperature values(s).
Can be a scalar, a sequence (e.g., list or tuple), a one-dimensional NumPy array, or a pandas Series.
Returns
-------
:py:obj:`~marine_qc.ValueIntType`
Same type as input, but with integer values
- Returns 2 (or array/sequence/Series of 2s) if either dpt or at2 is invalid (None or NaN).
- Returns 1 (or array/sequence/Series of 1s) if supersaturation is detected,
- Returns 0 (or array/sequence/Series of 0s) otherwise.
Raises
------
TypeError
If decorator `inspect_arrays` does not return np.ndarrays.
"""
dpt_arr, at2_arr = ensure_arrays(dpt=dpt, at2=at2)
result = np.full(dpt_arr.shape, untestable, dtype=int)
valid_indices = isvalid(dpt_arr) & isvalid(at2_arr)
cond_failed = np.full(dpt_arr.shape, True, dtype=bool)
cond_failed[valid_indices] = dpt_arr[valid_indices] > at2_arr[valid_indices]
result[valid_indices & cond_failed] = failed
result[valid_indices & ~cond_failed] = passed
return result
[docs]
@post_format_return_type(["sst"])
@inspect_arrays(["sst"])
@convert_units(sst="K", freezing_point="K")
def do_sst_freeze_check(
sst: ValueNumberType,
freezing_point: int | float,
freeze_check_n_sigma: int | float | Literal["default"] = "default",
sst_uncertainty: int | float | Literal["default"] = "default",
) -> ValueIntType:
"""
Check input sea-surface temperature(s) to see if it is above freezing.
This is a simple freezing point check made slightly more complex. We want to check if a
measurement of SST is above freezing, but there are two problems. First, the freezing point
can vary from place to place depending on the salinity of the water. Second, there is uncertainty
in SST measurements. If we place a hard cut-off at -1.8C, then we are likely to bias the average
of many measurements too high when they are near the freezing point - observational error will
push the measurements randomly higher and lower, and this test will trim out the lower tail, thus
biasing the result. The inclusion of an SST uncertainty parameter *might* mitigate that, and we allow
that possibility here. Note also that many ships make sea-surface temperature measurements to the nearest
whole degree, which in the case of water at or close to freezing would round to -2C and would fail a naive
test.
Parameters
----------
sst : :py:obj:`~marine_qc.ValueNumberType`
Input sea-surface temperature value(s) to be checked.
Can be a scalar, a sequence (e.g., list or tuple), a one-dimensional NumPy array, or a pandas Series.
freezing_point : float, optional
The freezing point of the water.
freeze_check_n_sigma : float, optional, default: "default"
Number of uncertainty standard deviations that sea surface temperature can be
below the freezing point before the QC check fails.
sst_uncertainty : float, optional, default: "default"
The uncertainty in the SST value.
Returns
-------
:py:obj:`~marine_qc.ValueIntType`
Same type as input, but with integer values
- Returns 2 (or array/sequence/Series of 2s) if any of `sst`, `freezing_point`, `sst_uncertainty`,
or `n_sigma` is numerically invalid (None or NaN).
- Returns 1 (or array/sequence/Series of 1s) if `sst` is below `freezing_point` by more than
`n_sigma` times `sst_uncertainty`.
- Returns 0 (or array/sequence/Series of 0s) otherwise.
Raises
------
TypeError
If decorator `inspect_arrays` does not return np.ndarrays.
Notes
-----
In previous versions, some parameters had default values:
* ``sst_uncertainty``: 0.0
* ``freezing_point``: -1.80
* ``n_sigma``: 2.0
"""
(sst_arr,) = ensure_arrays(sst=sst)
result = np.full(sst_arr.shape, untestable, dtype=int)
if not isvalid(sst_uncertainty) or not isvalid(freezing_point) or not isvalid(freeze_check_n_sigma):
return result
valid_sst = isvalid(sst_arr)
if freeze_check_n_sigma == "default":
freeze_check_n_sigma = 0.0
if sst_uncertainty == "default":
sst_uncertainty = 0.0
cond_failed = np.full(sst_arr.shape, True, dtype=bool)
cond_failed[valid_sst] = sst_arr[valid_sst] < (freezing_point - freeze_check_n_sigma * sst_uncertainty)
result[valid_sst & cond_failed] = failed
result[valid_sst & ~cond_failed] = passed
return result
[docs]
@post_format_return_type(["wind_speed", "wind_direction"])
@inspect_arrays(["wind_speed", "wind_direction"])
def do_wind_consistency_check(wind_speed: ValueNumberType, wind_direction: ValueNumberType) -> ValueIntType:
"""
Test to compare windspeed to winddirection to check if they are consistent.
Zero windspeed should correspond to no particular direction (variable) and
wind speeds above a threshold should correspond to a particular direction.
Parameters
----------
wind_speed : :py:obj:`~marine_qc.ValueNumberType`
Wind speed value(s).
Can be a scalar, a sequence (e.g., list or tuple), a one-dimensional NumPy array, or a pandas Series.
wind_direction : :py:obj:`~marine_qc.ValueNumberType`
Wind direction value(s).
Can be a scalar, a sequence (e.g., list or tuple), a one-dimensional NumPy array, or a pandas Series.
Returns
-------
:py:obj:`~marine_qc.ValueIntType`
Same type as input, but with integer values
- Returns 2 (or array/sequence/Series of 2s) if either wind_speed or wind_direction is invalid (None or NaN).
- Returns 1 (or array/sequence/Series of 1s) if wind_speed and wind_direction are inconsistent,
- Returns 0 (or array/sequence/Series of 0s) otherwise.
Raises
------
TypeError
If decorator `inspect_arrays` does not return np.ndarrays.
"""
wind_speed_arr, wind_direction_arr = ensure_arrays(wind_speed=wind_speed, wind_direction=wind_direction)
result = np.full(wind_speed_arr.shape, untestable, dtype=int)
valid_indices = isvalid(wind_speed_arr) & isvalid(wind_direction_arr)
cond_failed = np.full(wind_speed_arr.shape, True, dtype=bool)
cond_failed[valid_indices] = ((wind_speed_arr[valid_indices] == 0) & (wind_direction_arr[valid_indices] != 0)) | (
(wind_speed_arr[valid_indices] != 0) & (wind_direction_arr[valid_indices] == 0)
)
result[valid_indices & cond_failed] = failed
result[valid_indices & ~cond_failed] = passed
return result
def _do_mask_check(
lat: np.ndarray,
lon: np.ndarray,
mask: np.ndarray,
flag: int,
) -> np.ndarray:
"""
Check input position(s) to determine whether they correspond to a masked point.
Parameters
----------
lat : 1D np.ndarray of float
Latitude(s) of observation in degrees.
lon : 1D np.ndarray of float
Longitude() of observation in degree.
mask : 1D np.ndarray of int
Masked classification value(s) to which the latitude and longitude values(s) will be compared.
flag : int
Integer value in `mask` that denotes a specific point.
Returns
-------
:py:obj:`~marine_qc.ValueIntType`
Same type as input, but with integer values
- Returns 2 (or array/sequence/Series of 2s) if either latitude or longitude is numerically invalid (None/NaN).
- Returns 1 (or array/sequence/Series of 1s) if the position does not correspond to a land point
- Returns 0 (or array/sequence/Series of 0s) otherwise
Raises
------
ValueError
If decorator `inspect_arrays` does not return np.ndarrays.
"""
if mask.ndim == 0:
mask_arr = np.full_like(lat, mask)
else:
mask_arr = mask
result = np.full(lat.shape, untestable, dtype=int)
valid_indices = isvalid(lat) & isvalid(lon)
masked_points = np.full(lat.shape, True, dtype=bool)
masked_points[valid_indices] = mask_arr[valid_indices] == flag
result[valid_indices & masked_points] = passed
result[valid_indices & ~masked_points] = failed
return result
[docs]
@post_format_return_type(["lat", "lon"])
@inspect_arrays(["lat", "lon", "land_sea_mask"])
@convert_units(lat="degrees", lon="degrees")
@inspect_climatology("land_sea_mask")
def do_landlocked_check(
lat: ValueNumberType,
lon: ValueNumberType,
land_sea_mask: ClimArgType,
land_flag: int,
) -> ValueIntType:
"""
Check input position(s) to determine whether they correspond to a land point.
Parameters
----------
lat : :py:obj:`~marine_qc.ValueNumberType`
Latitude(s) of observation in degrees.
Can be a scalar, a sequence (e.g., list or tuple), a one-dimensional NumPy array, or a pandas Series.
lon : :py:obj:`~marine_qc.ValueNumberType`
Longitude() of observation in degree.
Can be a scalar, a sequence (e.g., list or tuple), a one-dimensional NumPy array, or a pandas Series.
land_sea_mask : :py:obj:`~marine_qc.ClimArgType`
Land-sea classification value(s) to which the latitude and longitude values(s) will be compared.
Can be a scalar, sequence, a one-dimensional NumPy array, a pandas Series,
:py:class:`~marine_qc.Climatology`, a path-like string on disk, a xarray Dataset or a xarray DataArray.
land_flag : int
Integer value in `land_sea_mask` that denotes a land point.
Returns
-------
:py:obj:`~marine_qc.ValueIntType`
Same type as input, but with integer values
- Returns 2 (or array/sequence/Series of 2s) if either latitude or longitude is numerically invalid (None/NaN).
- Returns 1 (or array/sequence/Series of 1s) if the position does not correspond to a land point
- Returns 0 (or array/sequence/Series of 0s) otherwise
Raises
------
ValueError
If decorator `inspect_arrays` does not return np.ndarrays.
"""
lat_arr, lon_arr, mask_arr = ensure_arrays(lat=lat, lon=lon, land_sea_mask=land_sea_mask)
return _do_mask_check(lat=lat_arr, lon=lon_arr, mask=mask_arr, flag=land_flag)
[docs]
@post_format_return_type(["lat", "lon"])
@inspect_arrays(["lat", "lon", "sea_land_mask"])
@convert_units(lat="degrees", lon="degrees")
@inspect_climatology("sea_land_mask")
def do_maritime_check(
lat: ValueNumberType,
lon: ValueNumberType,
sea_land_mask: ClimArgType,
sea_flag: int,
) -> ValueIntType:
"""
Check input position(s) to determine whether they correspond to a sea point.
Parameters
----------
lat : :py:obj:`~marine_qc.ValueNumberType`
Latitude(s) of observation in degrees.
Can be a scalar, a sequence (e.g., list or tuple), a one-dimensional NumPy array, or a pandas Series.
lon : :py:obj:`~marine_qc.ValueNumberType`
Longitude() of observation in degree.
Can be a scalar, a sequence (e.g., list or tuple), a one-dimensional NumPy array, or a pandas Series.
sea_land_mask : :py:obj:`~marine_qc.ClimArgType`
Sea-land classification value(s) to which the latitude and longitude values(s) will be compared.
Can be a scalar, sequence, a one-dimensional NumPy array, a pandas Series,
a :py:class:`~marine_qc.Climatology`, a path-like string on disk, a xarray Dataset or a xarray DataArray.
sea_flag : int
Integer value in `sea_land_mask` that denotes a sea point.
Returns
-------
:py:obj:`~marine_qc.ValueIntType`
Same type as input, but with integer values
- Returns 2 (or array/sequence/Series of 2s) if either latitude or longitude is numerically invalid (None/NaN).
- Returns 1 (or array/sequence/Series of 1s) if latitude and longitude denotes not a sea point
- Returns 0 (or array/sequence/Series of 0s) otherwise
Raises
------
ValueError
If decorator `inspect_arrays` does not return np.ndarrays.
"""
lat_arr, lon_arr, mask_arr = ensure_arrays(lat=lat, lon=lon, land_sea_mask=sea_land_mask)
return _do_mask_check(lat=lat_arr, lon=lon_arr, mask=mask_arr, flag=sea_flag)