{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "# Example: Compare ASCAT SM against ISMN" ] }, { "cell_type": "code", "execution_count": 4, "metadata": { "ExecuteTime": { "end_time": "2024-06-14T13:26:30.786613Z", "start_time": "2024-06-14T13:26:30.783029Z" } }, "outputs": [], "source": [ "# install the ISMN package first https://github.com/TUW-GEO/ismn\n", "import ismn.interface as ismn \n", "import warnings\n", "\n", "# install the ascat package first https://github.com/TUW-GEO/ascat\n", "from ascat.read_native.cdr import AscatGriddedNcTs\n", "\n", "import pytesmo.temporal_matching as temp_match\n", "import pytesmo.scaling as scaling\n", "import pytesmo.metrics as metrics\n", "from pytesmo.utils import rootdir\n", "\n", "import pandas as pd\n", "import matplotlib.pyplot as plt\n", "import tempfile\n", "\n", "%matplotlib inline" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Create the ascat reader:" ] }, { "cell_type": "code", "execution_count": 5, "metadata": { "ExecuteTime": { "end_time": "2024-06-14T13:26:31.054848Z", "start_time": "2024-06-14T13:26:30.818658Z" } }, "outputs": [], "source": [ "testdata_path = rootdir() / \"tests\" / \"test-data\"\n", "ascat_data_folder = testdata_path / \"sat\" / \"ascat\" / \"netcdf\" / \"55R22\"\n", "ascat_grid_fname = testdata_path / \"sat\" / \"ascat\" / \"netcdf\" / \"grid\" / \"TUW_WARP5_grid_info_2_1.nc\"\n", "static_layer_path = testdata_path / \"sat\" / \"h_saf\" / \"static_layer\"\n", "\n", "\n", "#init the AscatSsmCdr reader with the paths\n", "with warnings.catch_warnings():\n", " warnings.filterwarnings('ignore') # some warnings are expected and ignored\n", " \n", " ascat_reader = AscatGriddedNcTs(\n", " ascat_data_folder,\n", " \"TUW_METOP_ASCAT_WARP55R22_{:04d}\",\n", " grid_filename=ascat_grid_fname,\n", " static_layer_path=static_layer_path\n", " )" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Create the ismn reader:" ] }, { "cell_type": "code", "execution_count": 6, "metadata": { "scrolled": true, "ExecuteTime": { "end_time": "2024-06-14T13:26:31.231377Z", "start_time": "2024-06-14T13:26:31.055833Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Processing metadata for all ismn stations into folder /home/wpreimes/shares/home/code/pytesmo/tests/test-data/ismn/multinetwork/header_values.\n", "This may take a few minutes, but is only done once...\n", "Hint: Use `parallel=True` to speed up metadata generation for large datasets\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Files Processed: 100%|██████████| 8/8 [00:00<00:00, 64.84it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "Metadata generation finished after 0 Seconds.\n", "Metadata and Log stored in /tmp/tmpkfwxw4c4\n", "Found existing ismn metadata in /tmp/tmpkfwxw4c4/header_values.csv.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\n", "/home/wpreimes/miniforge3/envs/pytesmo/lib/python3.10/site-packages/ismn/filecollection.py:435: FutureWarning: Downcasting object dtype arrays on .fillna, .ffill, .bfill is deprecated and will change in a future version. Call result.infer_objects(copy=False) instead. To opt-in to the future behavior, set `pd.set_option('future.no_silent_downcasting', True)`\n", " dfs = dfs.fillna(np.nan)\n" ] }, { "data": { "text/plain": "[Station 'CST-01' with Sensors: ['ECH20-EC-TM_soil_moisture_0.050000_0.050000'],\n Station 'CST-02' with Sensors: ['ECH20-EC-TM_soil_moisture_0.050000_0.050000'],\n Station 'AAMU-jtg' with Sensors: ['Hydraprobe-Analog-(2.5-Volt)_soil_moisture_0.050000_0.050000'],\n Station 'Abrams' with Sensors: ['Hydraprobe-Analog-(2.5-Volt)_soil_moisture_0.050000_0.050000'],\n Station 'AdamsRanch#1' with Sensors: ['Hydraprobe-Analog-(2.5-Volt)_soil_moisture_0.050000_0.050000'],\n Station 'node414' with Sensors: ['EC5_soil_moisture_0.050000_0.050000'],\n Station 'node505' with Sensors: ['EC5_soil_moisture_0.050000_0.050000'],\n Station 'node703' with Sensors: ['EC5_soil_moisture_0.050000_0.050000']]" }, "execution_count": 6, "metadata": {}, "output_type": "execute_result" } ], "source": [ "#set path to ISMN data\n", "path_to_ismn_data = testdata_path / \"ismn\" / \"multinetwork\" / \"header_values\"\n", "\n", "#Initialize reader\n", "meta_path = tempfile.mkdtemp()\n", "ISMN_reader = ismn.ISMN_Interface(path_to_ismn_data, meta_path=meta_path)\n", "list(ISMN_reader.stations_that_measure('soil_moisture'))" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "We will compare only the first station to ASCAT here. For this station, we will compare ASCAT to the available measured time series from depths above 10 cm (which is only one in this case)." ] }, { "cell_type": "code", "execution_count": 7, "metadata": { "ExecuteTime": { "end_time": "2024-06-14T13:26:31.234536Z", "start_time": "2024-06-14T13:26:31.232239Z" } }, "outputs": [], "source": [ "station = next(ISMN_reader.stations_that_measure('soil_moisture'))" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "We will first temporally collocate the ISMN time series to ASCAT. Then we will perform a CDF matching so that biases between the two will be removed." ] }, { "cell_type": "code", "execution_count": 8, "metadata": { "ExecuteTime": { "end_time": "2024-06-14T13:26:37.149900Z", "start_time": "2024-06-14T13:26:31.236020Z" } }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "/home/wpreimes/miniforge3/envs/pytesmo/lib/python3.10/site-packages/ismn/filehandlers.py:638: FutureWarning: Support for nested sequences for 'parse_dates' in pd.read_csv is deprecated. Combine the desired columns with pd.to_datetime after parsing instead.\n", " return pd.read_csv(\n", "/home/wpreimes/miniforge3/envs/pytesmo/lib/python3.10/site-packages/ismn/filehandlers.py:638: FutureWarning: The 'delim_whitespace' keyword in pd.read_csv is deprecated and will be removed in a future version. Use ``sep='\\s+'`` instead\n", " return pd.read_csv(\n" ] } ], "source": [ "label_ascat='sm'\n", "label_insitu='insitu_sm'\n", "\n", "ISMN_time_series = station[0].read_data()\n", "ascat_time_series = ascat_reader.read(station.lon,\n", " station.lat,\n", " mask_ssf=True,\n", " mask_frozen_prob = 5,\n", " mask_snow_prob = 5).tz_localize(\"UTC\")" ] }, { "cell_type": "code", "execution_count": 9, "metadata": { "pycharm": { "name": "#%%\n" }, "ExecuteTime": { "end_time": "2024-06-14T13:26:37.169189Z", "start_time": "2024-06-14T13:26:37.150724Z" } }, "outputs": [ { "data": { "text/plain": " ssf proc_flag orbit_dir sm sm_noise \\\n2007-05-22 02:41:57.999995+00:00 1 0 b'D' 62.0 3.0 \n2007-05-22 14:00:06.000021+00:00 1 0 b'A' 87.0 4.0 \n2007-05-24 03:40:32.000005+00:00 1 0 b'D' 70.0 3.0 \n2007-05-25 14:37:59.999998+00:00 1 0 b'A' 87.0 4.0 \n2007-05-27 02:38:29.999996+00:00 1 0 b'D' 71.0 3.0 \n... ... ... ... ... ... \n2014-06-25 13:51:37.000006+00:00 1 0 b'A' 83.0 4.0 \n2014-06-27 03:32:04.999979+00:00 1 0 b'D' 79.0 4.0 \n2014-06-27 14:50:18.000019+00:00 1 0 b'A' 81.0 4.0 \n2014-06-30 02:30:05+00:00 1 0 b'D' 77.0 4.0 \n2014-06-30 13:48:17.999999+00:00 1 0 b'A' 83.0 4.0 \n\n snow_prob frozen_prob abs_sm_gldas \\\n2007-05-22 02:41:57.999995+00:00 0 4 NaN \n2007-05-22 14:00:06.000021+00:00 0 4 NaN \n2007-05-24 03:40:32.000005+00:00 0 4 NaN \n2007-05-25 14:37:59.999998+00:00 0 4 NaN \n2007-05-27 02:38:29.999996+00:00 0 4 NaN \n... ... ... ... \n2014-06-25 13:51:37.000006+00:00 0 0 NaN \n2014-06-27 03:32:04.999979+00:00 0 0 NaN \n2014-06-27 14:50:18.000019+00:00 0 0 NaN \n2014-06-30 02:30:05+00:00 0 0 NaN \n2014-06-30 13:48:17.999999+00:00 0 0 NaN \n\n abs_sm_noise_gldas abs_sm_hwsd \\\n2007-05-22 02:41:57.999995+00:00 NaN NaN \n2007-05-22 14:00:06.000021+00:00 NaN NaN \n2007-05-24 03:40:32.000005+00:00 NaN NaN \n2007-05-25 14:37:59.999998+00:00 NaN NaN \n2007-05-27 02:38:29.999996+00:00 NaN NaN \n... ... ... \n2014-06-25 13:51:37.000006+00:00 NaN NaN \n2014-06-27 03:32:04.999979+00:00 NaN NaN \n2014-06-27 14:50:18.000019+00:00 NaN NaN \n2014-06-30 02:30:05+00:00 NaN NaN \n2014-06-30 13:48:17.999999+00:00 NaN NaN \n\n abs_sm_noise_hwsd \n2007-05-22 02:41:57.999995+00:00 NaN \n2007-05-22 14:00:06.000021+00:00 NaN \n2007-05-24 03:40:32.000005+00:00 NaN \n2007-05-25 14:37:59.999998+00:00 NaN \n2007-05-27 02:38:29.999996+00:00 NaN \n... ... \n2014-06-25 13:51:37.000006+00:00 NaN \n2014-06-27 03:32:04.999979+00:00 NaN \n2014-06-27 14:50:18.000019+00:00 NaN \n2014-06-30 02:30:05+00:00 NaN \n2014-06-30 13:48:17.999999+00:00 NaN \n\n[830 rows x 11 columns]", "text/html": "
| \n | ssf | \nproc_flag | \norbit_dir | \nsm | \nsm_noise | \nsnow_prob | \nfrozen_prob | \nabs_sm_gldas | \nabs_sm_noise_gldas | \nabs_sm_hwsd | \nabs_sm_noise_hwsd | \n
|---|---|---|---|---|---|---|---|---|---|---|---|
| 2007-05-22 02:41:57.999995+00:00 | \n1 | \n0 | \nb'D' | \n62.0 | \n3.0 | \n0 | \n4 | \nNaN | \nNaN | \nNaN | \nNaN | \n
| 2007-05-22 14:00:06.000021+00:00 | \n1 | \n0 | \nb'A' | \n87.0 | \n4.0 | \n0 | \n4 | \nNaN | \nNaN | \nNaN | \nNaN | \n
| 2007-05-24 03:40:32.000005+00:00 | \n1 | \n0 | \nb'D' | \n70.0 | \n3.0 | \n0 | \n4 | \nNaN | \nNaN | \nNaN | \nNaN | \n
| 2007-05-25 14:37:59.999998+00:00 | \n1 | \n0 | \nb'A' | \n87.0 | \n4.0 | \n0 | \n4 | \nNaN | \nNaN | \nNaN | \nNaN | \n
| 2007-05-27 02:38:29.999996+00:00 | \n1 | \n0 | \nb'D' | \n71.0 | \n3.0 | \n0 | \n4 | \nNaN | \nNaN | \nNaN | \nNaN | \n
| ... | \n... | \n... | \n... | \n... | \n... | \n... | \n... | \n... | \n... | \n... | \n... | \n
| 2014-06-25 13:51:37.000006+00:00 | \n1 | \n0 | \nb'A' | \n83.0 | \n4.0 | \n0 | \n0 | \nNaN | \nNaN | \nNaN | \nNaN | \n
| 2014-06-27 03:32:04.999979+00:00 | \n1 | \n0 | \nb'D' | \n79.0 | \n4.0 | \n0 | \n0 | \nNaN | \nNaN | \nNaN | \nNaN | \n
| 2014-06-27 14:50:18.000019+00:00 | \n1 | \n0 | \nb'A' | \n81.0 | \n4.0 | \n0 | \n0 | \nNaN | \nNaN | \nNaN | \nNaN | \n
| 2014-06-30 02:30:05+00:00 | \n1 | \n0 | \nb'D' | \n77.0 | \n4.0 | \n0 | \n0 | \nNaN | \nNaN | \nNaN | \nNaN | \n
| 2014-06-30 13:48:17.999999+00:00 | \n1 | \n0 | \nb'A' | \n83.0 | \n4.0 | \n0 | \n0 | \nNaN | \nNaN | \nNaN | \nNaN | \n
830 rows × 11 columns
\n| \n | soil_moisture | \nsoil_moisture_flag | \nsoil_moisture_orig_flag | \n
|---|---|---|---|
| date_time | \n\n | \n | \n |
| 2008-07-01 00:00:00 | \n0.50 | \nC03 | \nM | \n
| 2008-07-01 01:00:00 | \n0.50 | \nC03 | \nM | \n
| 2008-07-01 02:00:00 | \n0.50 | \nC03 | \nM | \n
| 2008-07-01 03:00:00 | \n0.50 | \nC03 | \nM | \n
| 2008-07-01 04:00:00 | \n0.50 | \nC03 | \nM | \n
| ... | \n... | \n... | \n... | \n
| 2010-07-31 19:00:00 | \n0.26 | \nU | \nM | \n
| 2010-07-31 20:00:00 | \n0.26 | \nU | \nM | \n
| 2010-07-31 21:00:00 | \n0.26 | \nU | \nM | \n
| 2010-07-31 22:00:00 | \n0.26 | \nU | \nM | \n
| 2010-07-31 23:00:00 | \n0.26 | \nU | \nM | \n
15927 rows × 3 columns
\n| \n | sm | \ninsitu_sm | \n
|---|---|---|
| 2008-07-01 02:41:05+00:00 | \n79.0 | \n0.50 | \n
| 2008-07-01 13:59:12.999987+00:00 | \n81.0 | \n0.49 | \n
| 2008-07-02 13:38:37.000012+00:00 | \n60.0 | \n0.47 | \n
| 2008-07-03 03:39:39.000011+00:00 | \n68.0 | \n0.46 | \n
| 2008-07-04 14:37:08.999993+00:00 | \n66.0 | \n0.45 | \n
| ... | \n... | \n... | \n
| 2010-07-25 13:57:54.000013+00:00 | \n78.0 | \n0.41 | \n
| 2010-07-27 03:38:19.999997+00:00 | \n76.0 | \n0.38 | \n
| 2010-07-28 14:35:46.999995+00:00 | \n67.0 | \n0.35 | \n
| 2010-07-30 02:36:15.000004+00:00 | \n66.0 | \n0.31 | \n
| 2010-07-30 13:54:24.999980+00:00 | \n63.0 | \n0.30 | \n
244 rows × 2 columns
\n