diff --git a/binder/CallipsoL2_access.ipynb b/binder/CallipsoL2_access.ipynb index 90dae80..b1f733e 100644 --- a/binder/CallipsoL2_access.ipynb +++ b/binder/CallipsoL2_access.ipynb @@ -101,10 +101,14 @@ }, "outputs": [], "source": [ - "auth = earthaccess.login(strategy=\"netrc\", persist=True) # you will be promted to add your EDL credentials\n", + "from earthaccess.exceptions import LoginStrategyUnavailable\n", + "try:\n", + " auth = earthaccess.login(strategy=\"netrc\", persist=True) # you will be promted to add your EDL credentials\n", + "except LoginStrategyUnavailable:\n", + " auth = earthaccess.login(strategy=\"interactive\", persist=True)\n", "\n", "# pass Token Authorization to a new Session.\n", - "my_session = session=auth.get_session()" + "my_session = session=auth.get_session()\n" ] }, { diff --git a/binder/DAYMET.ipynb b/binder/DAYMET.ipynb index 5572537..c9cdd5c 100644 --- a/binder/DAYMET.ipynb +++ b/binder/DAYMET.ipynb @@ -64,10 +64,14 @@ "metadata": {}, "outputs": [], "source": [ - "auth = earthaccess.login(strategy=\"netrc\", persist=True) # \n", + "from earthaccess.exceptions import LoginStrategyUnavailable\n", + "try:\n", + " auth = earthaccess.login(strategy=\"netrc\", persist=True) # you will be promted to add your EDL credentials\n", + "except LoginStrategyUnavailable:\n", + " auth = earthaccess.login(strategy=\"interactive\", persist=True)\n", "\n", "# pass Token Authorization to a new Session.\n", - "my_session = auth.get_session()" + "my_session = session=auth.get_session()\n" ] }, { diff --git a/binder/Ecostress.ipynb b/binder/Ecostress.ipynb index dfeba54..eef98c8 100644 --- a/binder/Ecostress.ipynb +++ b/binder/Ecostress.ipynb @@ -8,6 +8,17 @@ "# Accessing ECOSTRESS data" ] }, + { + "cell_type": "code", + "execution_count": null, + "id": "c5b07318-af7b-4470-b02d-a0ead104b568", + "metadata": {}, + "outputs": [], + "source": [ + "%load_ext autoreload\n", + "%autoreload 2" + ] + }, { "cell_type": "code", "execution_count": null, @@ -43,10 +54,14 @@ "metadata": {}, "outputs": [], "source": [ - "auth = earthaccess.login(strategy=\"netrc\", persist=True) # you will be promted to add your EDL credentials\n", + "from earthaccess.exceptions import LoginStrategyUnavailable\n", + "try:\n", + " auth = earthaccess.login(strategy=\"netrc\", persist=True) # you will be promted to add your EDL credentials\n", + "except LoginStrategyUnavailable:\n", + " auth = earthaccess.login(strategy=\"interactive\", persist=True)\n", "\n", "# pass Token Authorization to a new Session.\n", - "my_session = session=auth.get_session()" + "my_session = session=auth.get_session()\n" ] }, { @@ -68,7 +83,7 @@ "source": [ "ECOSTRESS_ccid = \"C2076114664-LPCLOUD\"\n", "bounding_box = [-128.847656,41.112469,-107.050781,46.679594]\n", - "time_range = [dt.datetime(2025, 3, 1), dt.datetime(2025, 3, 14)]\n", + "time_range = [dt.datetime(2025, 3, 1), dt.datetime(2025, 3, 30)]\n", "\n", "\n", "cmr_urls = get_cmr_urls(ccid=ECOSTRESS_ccid, bounding_box = bounding_box, limit=1000, time_range=time_range) # limit by default = 50\n", @@ -91,21 +106,38 @@ ] }, { - "cell_type": "code", - "execution_count": null, + "cell_type": "markdown", "id": "7744670c-cc38-4f92-a6bc-65e640657e8f", "metadata": { "tags": [] }, + "source": [ + "### Subset data\n", + "\n", + "First, we would like to subset data based on the criteria:\n", + "\n", + "* Daylight data\n", + "* QAPercentCloudCover <30% \n", + "* and QAPercentGoodQuality> 70%\n", + "\n", + "\n", + "For that, we need to download only 3 variables:\n", + "\n", + "```python\n", + " [\"/L2 LSTE Metadata/QAPercentCloudCover\", \n", + " \"/L2 LSTE Metadata/QAPercentGoodQuality\",\n", + " \"/StandardMetadata/DayNightFlag\"]\n", + "```" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "024bd204-f36c-4249-acce-f36d25aeb736", + "metadata": {}, "outputs": [], "source": [ - "keep_vars = ['/SDS/LST',\n", - " \"/StandardMetadata/EastBoundingCoordinate\",\n", - " \"/StandardMetadata/SouthBoundingCoordinate\",\n", - " \"/StandardMetadata/NorthBoundingCoordinate\",\n", - " \"/StandardMetadata/DayNightFlag\",\n", - " \"/StandardMetadata/WestBoundingCoordinate\",\n", - "]" + "output_path = 'data/'" ] }, { @@ -123,36 +155,82 @@ { "cell_type": "code", "execution_count": null, - "id": "f6936bb4-ed79-49e2-b70f-1f1256a60b78", - "metadata": {}, + "id": "5fb47fd7-0743-4723-979b-e97ce2068303", + "metadata": { + "tags": [] + }, "outputs": [], "source": [ - "output_path = \"data/\"" + "%%time\n", + "dap_to_netcdf(cmr_urls, session=my_session, output_path = output_path, \n", + " keep_variables=[\"/L2 LSTE Metadata/QAPercentCloudCover\", \n", + " \"/L2 LSTE Metadata/QAPercentGoodQuality\",\n", + " \"/StandardMetadata/DayNightFlag\"]\n", + " )" + ] + }, + { + "cell_type": "markdown", + "id": "c6c4b77f-7fa2-43e3-aed4-940d8c5d183d", + "metadata": {}, + "source": [ + "## Inspect local files\n", + "\n", + "Data has been already downloaded, and we can further filter to identify remote\n", + "granules by\n", + "\n", + "* Daylight data\n", + "* QAPercentCloudCover <30% \n", + "* and QAPercentGoodQuality> 70%\n", + "\n", + "Then we will update our list of URLs to download from, based on the remote files that\n", + "satisfy the above criteria\n", + "\n", + "**NOTE**: The files cannot be aggregated!!\n" ] }, { "cell_type": "code", "execution_count": null, - "id": "5fb47fd7-0743-4723-979b-e97ce2068303", - "metadata": { - "tags": [] - }, + "id": "77875486-b7e4-4f92-8929-b817dbef5244", + "metadata": {}, "outputs": [], "source": [ - "%%time\n", - "dap_to_netcdf(cmr_urls, session=my_session, output_path = output_path, keep_variables=keep_vars)" + "final_urls = []\n", + "for i in range(len(cmr_urls)):\n", + " local_file = output_path+cmr_urls[i].split(\"/\")[-1]+\".nc4\"\n", + " dst = xr.open_datatree(local_file).load()\n", + " if dst['L2 LSTE Metadata/QAPercentCloudCover'].values < 30 and dst[\"L2 LSTE Metadata/QAPercentGoodQuality\"] > 70 and dst[\"/StandardMetadata/DayNightFlag\"] == 'Day':\n", + " final_urls.append(cmr_urls[i])\n", + "\n", + "print(\"Total remote granules to download: \", len(final_urls))" ] }, { "cell_type": "code", "execution_count": null, - "id": "a2765385-8703-4536-93ad-b3d9b6daba57", - "metadata": { - "tags": [] - }, + "id": "555ee69e-3ab1-4f68-8171-02da058a654c", + "metadata": {}, "outputs": [], "source": [] }, + { + "cell_type": "markdown", + "id": "fabf994b-a1e7-4982-ac9b-046a04ef760d", + "metadata": {}, + "source": [ + "### Declare final variables to download\n", + "\n", + "At this point, it is crucial to remove any ECOSTRESS data downloaded into the\n", + "data_output directory. For that open a terminal and navigate to the data directory\n", + "\n", + "\n", + "```shell\n", + "cd data\n", + "rm ECOv002_L2*.nc4\n", + "```" + ] + }, { "cell_type": "code", "execution_count": null, @@ -160,20 +238,58 @@ "metadata": {}, "outputs": [], "source": [ - "dt = xr.open_datatree(\"data/ECOv002_L2_LSTE_37723_006_20250302T070023_0713_01.nc4\")\n", - "dt" + "# define all variables of interest that our final download will have\n", + "\n", + "keep_vars = [\"/StandardMetadata/EastBoundingCoordinate\", \n", + " \"/StandardMetadata/SouthBoundingCoordinate\", \n", + " \"/StandardMetadata/NorthBoundingCoordinate\",\n", + " \"/StandardMetadata/WestBoundingCoordinate\",\n", + " \"/StandardMetadata/DayNightFlag\",\n", + " \"/StandardMetadata/ImagePixels\",\n", + " \"/StandardMetadata/ImagePixelSpacing\",\n", + " \"/StandardMetadata/ImageLines\",\n", + " \"/StandardMetadata/RangeBeginningDate\",\n", + " \"/StandardMetadata/RangeBeginningTime\",\n", + " \"/StandardMetadata/RangeEndingDate\",\n", + " \"/L2 LSTE Metadata/QAPercentCloudCover\",\n", + " \"/L2 LSTE Metadata/QAPercentGoodQuality\",\n", + " \"/SDS/QC\", \"/SDS/LST\",\n", + " ]" + ] + }, + { + "cell_type": "markdown", + "id": "41c429de-1bab-4a79-8fea-6055073bbf13", + "metadata": {}, + "source": [ + "### Data Download\n", + "\n", + "Do not forget to now use the final_urls list, which points to our granules of interest.\n" ] }, { "cell_type": "code", "execution_count": null, - "id": "30c713c6-cee4-4ba4-8b4c-1edb8ee09583", - "metadata": { - "tags": [] - }, + "id": "a17ec9bb-ecb4-4c9f-b198-32a484f1a225", + "metadata": {}, "outputs": [], "source": [ - "dt['SDS/LST'].isel(dim0=slice(0, 1000), dim1=slice(0, 1400)).plot()" + "%%time\n", + "dap_to_netcdf(final_urls, session=my_session, output_path = output_path, \n", + " keep_variables=keep_vars,\n", + " )" + ] + }, + { + "cell_type": "markdown", + "id": "5c60ff18-746f-4df0-925e-542fb81007e1", + "metadata": {}, + "source": [ + "### Inspect local files\n", + "\n", + "Data has been already downloaded! \n", + "\n", + "**NOTE:** This datasets cannot be aggregated\n" ] }, { @@ -182,6 +298,58 @@ "id": "7b095687-800d-48c2-a1cc-50cd8f733fb0", "metadata": {}, "outputs": [], + "source": [ + "local_file = output_path+final_urls[0].split(\"/\")[-1]+\".nc4\"\n", + "dst = xr.open_datatree(local_file)\n", + "dst\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c3ce3ff9-2e1d-4f8a-a164-772dc9f6695a", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "330aac77-5748-4875-beb9-8a9564bca085", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b9e25760-4361-48a3-9e50-e2a1a1164359", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0128a980-ea83-48b4-ad0e-4e6eed0b26f2", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "54f32f9d-2731-4cc9-9685-1e63582afbd3", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "15276546-459e-4dc1-a7b8-5fc0140d24f2", + "metadata": {}, + "outputs": [], "source": [] } ], diff --git a/binder/MERRA2.ipynb b/binder/MERRA2.ipynb index c58cab9..0fbc19f 100644 --- a/binder/MERRA2.ipynb +++ b/binder/MERRA2.ipynb @@ -97,10 +97,14 @@ }, "outputs": [], "source": [ - "auth = earthaccess.login(strategy=\"netrc\", persist=True) # you will be promted to add your EDL credentials\n", + "from earthaccess.exceptions import LoginStrategyUnavailable\n", + "try:\n", + " auth = earthaccess.login(strategy=\"netrc\", persist=True) # you will be promted to add your EDL credentials\n", + "except LoginStrategyUnavailable:\n", + " auth = earthaccess.login(strategy=\"interactive\", persist=True)\n", "\n", "# pass Token Authorization to a new Session.\n", - "my_session = auth.get_session()" + "my_session = session=auth.get_session()\n" ] }, { diff --git a/binder/PACEL3_Chrol_a.ipynb b/binder/PACEL3_Chrol_a.ipynb index b9d463b..b253a16 100644 --- a/binder/PACEL3_Chrol_a.ipynb +++ b/binder/PACEL3_Chrol_a.ipynb @@ -94,14 +94,6 @@ "chlor_a_urls[:5]" ] }, - { - "cell_type": "code", - "execution_count": null, - "id": "8fce82b9-a48f-40f6-8fbd-a4be6c67473f", - "metadata": {}, - "outputs": [], - "source": [] - }, { "cell_type": "markdown", "id": "d8e45e61-896d-456e-9e0c-9e0b504b62da", @@ -123,10 +115,14 @@ }, "outputs": [], "source": [ - "auth = earthaccess.login(strategy=\"netrc\", persist=True) # you will be promted to add your EDL credentials\n", + "from earthaccess.exceptions import LoginStrategyUnavailable\n", + "try:\n", + " auth = earthaccess.login(strategy=\"netrc\", persist=True) # you will be promted to add your EDL credentials\n", + "except LoginStrategyUnavailable:\n", + " auth = earthaccess.login(strategy=\"interactive\", persist=True)\n", "\n", "# pass Token Authorization to a new Session.\n", - "my_session = session=auth.get_session()" + "my_session = session=auth.get_session()\n" ] }, { diff --git a/binder/SWOT.ipynb b/binder/SWOT.ipynb index 1821207..96f1f5c 100644 --- a/binder/SWOT.ipynb +++ b/binder/SWOT.ipynb @@ -10,6 +10,17 @@ "# Access SWOT Level 2 data with OPeNDAP" ] }, + { + "cell_type": "code", + "execution_count": null, + "id": "c22451a7-1f0a-4b35-b947-5c6e276707e2", + "metadata": {}, + "outputs": [], + "source": [ + "%load_ext autoreload\n", + "%autoreload 2" + ] + }, { "cell_type": "code", "execution_count": null, @@ -24,7 +35,8 @@ "import earthaccess\n", "# import pydap-specific tools\n", "from pydap.client import open_url,get_cmr_urls\n", - "from pydap.client import to_netcdf as dap_to_netcdf" + "from pydap.client import to_netcdf as dap_to_netcdf\n", + "import numpy as np" ] }, { @@ -49,10 +61,10 @@ "outputs": [], "source": [ "swot_ccid = \"C2799438313-POCLOUD\"\n", - "time_range = [dt.datetime(2023, 2, 1), dt.datetime(2023, 2, 12)] # One month of data\n", + "time_range = [dt.datetime(2023, 1, 1), dt.datetime(2023, 1, 31)] # One month of data\n", "\n", "# select a region in the Iceland-Faroe Ridge\n", - "bbox = [-20.760731,60.080727, -4.297294,65.675099] #[west, south, east, north]\n", + "bbox = [-20.760731, 60.080727, -4.297294,65.675099] #[west, south, east, north]\n", "\n", "cmr_urls = get_cmr_urls(ccid=swot_ccid, bounding_box = bbox, time_range=time_range, limit=1000) # you can incread the limit of results\n", "print(\"################################################ \\n We found a total of \", len(cmr_urls), \"OPeNDAP URLS!!!\\n################################################\")" @@ -79,10 +91,14 @@ }, "outputs": [], "source": [ - "auth = earthaccess.login(strategy=\"netrc\", persist=True) # you will be promted to add your EDL credentials\n", + "from earthaccess.exceptions import LoginStrategyUnavailable\n", + "try:\n", + " auth = earthaccess.login(strategy=\"netrc\", persist=True) # you will be promted to add your EDL credentials\n", + "except LoginStrategyUnavailable:\n", + " auth = earthaccess.login(strategy=\"interactive\", persist=True)\n", "\n", "# pass Token Authorization to a new Session.\n", - "my_session=auth.get_session()" + "my_session = session=auth.get_session()\n" ] }, { @@ -102,6 +118,27 @@ "\n" ] }, + { + "cell_type": "code", + "execution_count": null, + "id": "b91ecea1-1dee-4f23-a61d-4edffd33416b", + "metadata": {}, + "outputs": [], + "source": [ + "swot_standard_urls = [url for url in cmr_urls if url.split('/')[-1][:8]=='SWOT_GPN']\n", + "len(swot_standard_urls)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d3d85071-5ea2-484e-897e-d5b0d5301672", + "metadata": {}, + "outputs": [], + "source": [ + "cmr_urls[:6]" + ] + }, { "cell_type": "code", "execution_count": null, @@ -112,35 +149,44 @@ "outputs": [], "source": [ "%%time\n", - "pyds = open_url(cmr_urls[0], protocol=\"dap4\", session=my_session)\n", + "pyds = open_url(swot_standard_urls[0], protocol=\"dap4\", session=my_session)\n", "pyds.tree()" ] }, { "cell_type": "code", "execution_count": null, - "id": "c76dbc9d-8481-4de8-8131-5833281c019e", - "metadata": { - "tags": [] - }, + "id": "95892a73-abf6-4ad0-983d-17a47efa5c8e", + "metadata": {}, "outputs": [], "source": [ - "Variables = [\n", - " \"/data_01/time\",\"/data_01/longitude\", \"/data_01/latitude\", \n", - " \"/data_01/surface_classification_flag\", \"/data_01/ice_flag\",\n", - " \"/data_01/mean_dynamic_topography\", \"/data_01/rain_flag\"\n", - "]\n", - "output_path = \"data/\"" + "output_path = \"data/\"\n" + ] + }, + { + "cell_type": "markdown", + "id": "320a6e22-82fb-4ab7-aada-9497893d0b74", + "metadata": {}, + "source": [ + "# Download only coordinate data\n", + "\n", + "This step will help identify the subset of most of data \n", + "is downloaded within our bounding box" ] }, { "cell_type": "code", "execution_count": null, - "id": "15d43ed9-8f59-43d6-b07a-b8ce43c615fa", - "metadata": {}, + "id": "c76dbc9d-8481-4de8-8131-5833281c019e", + "metadata": { + "tags": [] + }, "outputs": [], "source": [ - "Variables" + "Variables = [\n", + " \"/data_01/time\", \"/data_01/longitude\",\n", + " \"/data_20/time\",\"/data_20/longitude\",\n", + "]" ] }, { @@ -148,9 +194,8 @@ "id": "80dadd83-e736-4983-ba70-fc9c4192f639", "metadata": {}, "source": [ - "# Download all data\n", - "\n", - "Stream each remote file into its own local file, downloading only the data of interest" + "# Download data\n", + "\n" ] }, { @@ -163,7 +208,13 @@ "outputs": [], "source": [ "%%time\n", - "dap_to_netcdf(cmr_urls[:20], session=my_session, keep_variables=Variables, output_path=output_path)" + "dap_to_netcdf(swot_standard_urls, \n", + " session=my_session, \n", + " keep_variables= [\n", + " \"/data_01/time\", \"/data_01/longitude\",\n", + " \"/data_20/time\",\"/data_20/longitude\",\n", + " ],\n", + " output_path=output_path)" ] }, { @@ -181,23 +232,120 @@ "metadata": {}, "outputs": [], "source": [ - "dt1 = xr.open_datatree(output_path+f\"{cmr_urls[0].split('/')[-1]}.nc4\")" + "## get min lat/lon from \n", + "minLon, maxLon = bbox[0], bbox[2]\n", + "slices = []\n", + "for url in swot_standard_urls:\n", + " filename = output_path+f\"{url.split('/')[-1]}.nc4\"\n", + " dt1 = xr.open_datatree(filename).load()\n", + " # find index /data_01/longitude\n", + " longitude = dt1['data_01/longitude']\n", + " longitude = longitude.where(longitude <=180, longitude -360)\n", + " mask = (longitude >= minLon) & (longitude <= maxLon)\n", + " idx = np.nonzero(mask.values)[0]\n", + " # find index /data_20/longitude\n", + " longitude = dt1['data_20/longitude']\n", + " longitude = longitude.where(longitude <=180, longitude -360)\n", + " mask = (longitude >= minLon) & (longitude <= maxLon)\n", + " idx2 = np.nonzero(mask.values)[0]\n", + " # create slice for each granule\n", + " slices.append({\"/data_01/time\":(idx[0],idx[-1]), \n", + " \"/data_20/time\": (idx2[0],idx2[-1])})" + ] + }, + { + "cell_type": "markdown", + "id": "a427f982-b69a-4714-ae05-32324a882a93", + "metadata": {}, + "source": [ + "## Declare All Variables of interest \n", + "\n", + "Using their Fully Qualifying Name" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "749ea54e-2761-477b-a4e7-3af482335a72", + "metadata": {}, + "outputs": [], + "source": [ + "Variables = [\n", + " \"/data_01/time\", \"/data_01/longitude\", \"/data_01/latitude\", \n", + " \"/data_01/sst\", \"/data_01/mean_dynamic_topography_interp_qual\",\n", + " \"/data_01/surface_classification_flag\", \"/data_01/ice_flag\",\n", + " \"/data_01/mean_dynamic_topography\", \"/data_01/rain_flag\",\n", + " \"/data_20/time\", \"/data_20/longitude\", \"/data_20/latitude\", \n", + " \"/data_20/distance_to_coast\",\n", + "]\n" + ] + }, + { + "cell_type": "markdown", + "id": "19bd8790-9c1c-4a4f-9127-1e3631f8549d", + "metadata": {}, + "source": [ + "## Download step\n", + "\n", + "Finally we download all data of interest, minimizing the data\n", + "outside of our bounding box\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "95a084a3-1e26-4f07-bea4-a7de7ae13f07", + "metadata": {}, + "outputs": [], + "source": [ + "%%time\n", + "dap_to_netcdf(swot_standard_urls, \n", + " session=my_session, \n", + " keep_variables=Variables, \n", + " dim_slices = slices, \n", + " output_path=output_path)" + ] + }, + { + "cell_type": "markdown", + "id": "045b0bc2-d3c2-49d3-9e02-e98c532bd486", + "metadata": {}, + "source": [ + "## Inspect a file" ] }, { "cell_type": "code", "execution_count": null, - "id": "0e1b336f-511c-4cba-ba3a-6bbf767f1e26", + "id": "9f2281a0-0330-470a-be65-dc8b1d38099a", "metadata": {}, "outputs": [], "source": [ + "filename = output_path+f\"{swot_standard_urls[0].split('/')[-1]}.nc4\"\n", + "dt1 = xr.open_datatree(filename).load()\n", "dt1" ] }, { "cell_type": "code", "execution_count": null, - "id": "9ed5821a-97fa-4ea3-b0f6-7620af68bcf2", + "id": "1782dc03-cc3d-4f4e-afcc-cc74533b1c3c", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e95ef821-8c53-4a3e-a092-993a573b44a5", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "10e60ea4-ce44-4ee6-ae05-772c16da6897", "metadata": {}, "outputs": [], "source": [] diff --git a/binder/TEMPO_L3.ipynb b/binder/TEMPO_L3.ipynb index cd4aec5..453d46a 100644 --- a/binder/TEMPO_L3.ipynb +++ b/binder/TEMPO_L3.ipynb @@ -105,10 +105,14 @@ "metadata": {}, "outputs": [], "source": [ - "auth = earthaccess.login(strategy=\"netrc\", persist=True) # you will be promted to add your EDL credentials\n", + "from earthaccess.exceptions import LoginStrategyUnavailable\n", + "try:\n", + " auth = earthaccess.login(strategy=\"netrc\", persist=True) # you will be promted to add your EDL credentials\n", + "except LoginStrategyUnavailable:\n", + " auth = earthaccess.login(strategy=\"interactive\", persist=True)\n", "\n", "# pass Token Authorization to a new Session.\n", - "my_session = session=auth.get_session()" + "my_session = session=auth.get_session()\n" ] }, { diff --git a/binder/TEMPO_NRT.ipynb b/binder/TEMPO_NRT.ipynb index 5d62a8c..db81c22 100644 --- a/binder/TEMPO_NRT.ipynb +++ b/binder/TEMPO_NRT.ipynb @@ -11,9 +11,6 @@ "\n", " **Requirements**\n", "1. EDL authentication (username/password)\n", - "2. Generate a Bearer Token.\n", - "4. Get `environment.yml` file and install conda environment to run notebook.\n", - "\n", "\n", " **Objectives**\n", "\n", @@ -45,7 +42,8 @@ "# import pydap-specific tools\n", "from pydap.net import create_session\n", "from pydap.client import get_cmr_urls, open_url\n", - "from pydap.client import to_netcdf as dap_to_netcdf" + "from pydap.client import to_netcdf as dap_to_netcdf\n", + "import numpy as np" ] }, { @@ -89,7 +87,7 @@ }, "outputs": [], "source": [ - "cmr_urls[0]\n" + "cmr_urls[:5]" ] }, { @@ -97,7 +95,7 @@ "id": "6e04deee-6c59-4437-9aed-381a73085496", "metadata": {}, "source": [ - " **EDL Authentication via OPeNDAP**\n", + " **EDL Authentication via earthaccess and OPeNDAP**\n", "\n", " You can authenticate via:\n", "\n", @@ -117,10 +115,14 @@ }, "outputs": [], "source": [ - "auth = earthaccess.login(strategy=\"netrc\", persist=True) # you will be promted to add your EDL credentials\n", + "from earthaccess.exceptions import LoginStrategyUnavailable\n", + "try:\n", + " auth = earthaccess.login(strategy=\"netrc\", persist=True) # you will be promted to add your EDL credentials\n", + "except LoginStrategyUnavailable:\n", + " auth = earthaccess.login(strategy=\"interactive\", persist=True)\n", "\n", "# pass Token Authorization to a new Session.\n", - "my_session = session=auth.get_session()" + "my_session = session=auth.get_session()\n" ] }, { @@ -128,18 +130,9 @@ "id": "a2322093-b9b6-49c5-8bb4-7ad17bffb95d", "metadata": {}, "source": [ - "# Accessing Metadata\n", - "\n", - " What are some tools, their differences, and what do they do.\n", - "\n", - " **PYDAP: Metadata-Only**\n", + "## Inspect Metadata\n", "\n", - "\n", - "\n", - "```{note}\n", - "Q: How do we tell the server which protocol to use?\n", - "A: By replaing the http -> dap4 in the URL\n", - "```\n" + " We can access OPeNDAP produced metadata to identify the variables of interest. In particular those associated with latitude and longitude values\n" ] }, { @@ -152,10 +145,30 @@ "outputs": [], "source": [ "%%time\n", - "pyds = open_url(cmr_urls[0], protocol=\"dap4\", session=my_session)\n", + "pyds = open_url(cmr_urls[0],protocol=\"dap4\", session=my_session)\n", "pyds.tree()" ] }, + { + "cell_type": "code", + "execution_count": null, + "id": "981a1235-1e86-4177-9ee3-142cfd614404", + "metadata": {}, + "outputs": [], + "source": [ + "dims = list(set(pyds['geolocation/latitude'].dims + pyds['geolocation/longitude'].dims + pyds['geolocation/time'].dims))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "93a8df34-91dc-4eec-b20e-66c5811a04c3", + "metadata": {}, + "outputs": [], + "source": [ + "print(\"\\nnecessary dimensions to download:\", dims, \"\\n\")" + ] + }, { "cell_type": "markdown", "id": "4510032f-11c6-4dc4-ba8a-0631f2ab2aef", @@ -170,9 +183,9 @@ "tags": [] }, "source": [ - "### Subset by Variables\n", + "### Subset by Variable Names\n", "\n", - "We will retain `xtrack`, and all data within Groups `product` and `geolocation`\n" + " First we explore only the coordinate and their dimensions, to identify spatial subset." ] }, { @@ -184,12 +197,7 @@ }, "outputs": [], "source": [ - "output_path = \"data/\"\n", - "\n", - "keep_variables = [\"/xtrack\",\"/mirror_step\", \n", - " \"/product\", \"/geolocation/time\",\n", - " \"/geolocation/longitude\",\"/geolocation/latitude\", \n", - " ]\n" + "output_path = \"data/\"" ] }, { @@ -197,9 +205,9 @@ "id": "ca68ec34-eea8-4984-8eb5-e56eb627314b", "metadata": {}, "source": [ - "# Stream data\n", + "## Stream data\n", "\n", - "each remote file is stored into an individual file. No data aggregation\n" + " Each remote file is stored into an individual file. No data aggregation\n" ] }, { @@ -212,57 +220,198 @@ "outputs": [], "source": [ "%%time\n", - "dap_to_netcdf(cmr_urls, session=my_session, keep_variables=keep_variables, output_path=output_path)" + "dap_to_netcdf(cmr_urls, session=my_session, \n", + " keep_variables= dims + [\n", + " \"/geolocation/time\",\n", + " \"/geolocation/longitude\",\n", + " \"/geolocation/latitude\", \n", + " ],\n", + " output_path=output_path)" + ] + }, + { + "cell_type": "markdown", + "id": "2f07b38f-b8b9-44bb-986c-d579739d8469", + "metadata": {}, + "source": [ + "### Inspect local data" ] }, { "cell_type": "code", "execution_count": null, - "id": "8093919a-6b92-494b-91d6-0f8247819616", - "metadata": { - "tags": [] - }, + "id": "b7aa7be3-9d88-4883-b2ef-625f7b79810c", + "metadata": {}, "outputs": [], "source": [ "%%time\n", - "dt = xr.open_datatree(output_path+\"TEMPO_NO2_L2_V04_20251001T141426Z_S004G08.nc4\")\n", - "dt['geolocation'].load()" + "# Use coord data from Bounding Box\n", + "minLon, maxLon = bounding_box[0], bounding_box[2]\n", + "minLat, maxLat = bounding_box[1], bounding_box[3]\n", + "\n", + "slices = []\n", + "# iterate over all downloaded files\n", + "# Will use the URL to extract the filename\n", + "for url in cmr_urls:\n", + " filename = output_path+f\"{url.split('/')[-1][:-3]}.nc4\"\n", + " # Flatten data \n", + " ds = xr.merge([xr.open_dataset(filename), xr.open_dataset(filename, group='geolocation')])\n", + " ds.load()\n", + " # Identify subset from Lon/Lat data per granule\n", + " \n", + " longitude = ds['longitude'].values\n", + " latitude = ds['latitude'].values\n", + "\n", + " mask = (\n", + " (longitude >= minLon) & (longitude <= maxLon) &\n", + " (latitude >= minLat) & (latitude <= maxLat)\n", + " )\n", + "\n", + " rows, cols = np.where(mask)\n", + " # indexes below\n", + " y0, y1 = rows.min(), rows.max()\n", + " x0, x1 = cols.min(), cols.max()\n", + " slice_ = {\n", + " \"mirror_step\":(y0,y1),\n", + " \"xtrack\": (x0,x1),\n", + " }\n", + " slices.append({\n", + " \"mirror_step\":(y0,y1),\n", + " \"xtrack\": (x0,x1),\n", + " })" + ] + }, + { + "cell_type": "markdown", + "id": "e3619957-e39b-4ea6-96c4-647d4ec5add9", + "metadata": {}, + "source": [ + "### Visualize Coordinates\n", + "\n", + " Will need to mask arrays for visualizing, \n", + "\n", + " Plot only the last granule" ] }, { "cell_type": "code", "execution_count": null, - "id": "e99e3954-116e-4ea0-867e-256cacd6be8f", - "metadata": { - "tags": [] - }, + "id": "5ab28256-3c89-4b00-a3f1-e3c924494fa7", + "metadata": {}, "outputs": [], "source": [ - "dt" + "Lon = ds['longitude'].isel(mirror_step=slice(y0, y1), xtrack=slice(x0, x1))\n", + "\n", + "Lon_masked = xr.full_like(ds['longitude'], np.nan)\n", + "Lon_masked.loc[dict(\n", + " mirror_step=Lon['mirror_step'],\n", + " xtrack=Lon['xtrack']\n", + ")] = Lon\n", + "\n", + "\n", + "Lat = ds['latitude'].isel(mirror_step=slice(y0, y1), xtrack=slice(x0, x1))\n", + "Lat_masked = xr.full_like(ds['latitude'], np.nan)\n", + "Lat_masked.loc[dict(\n", + " mirror_step=Lat['mirror_step'],\n", + " xtrack=Lat['xtrack']\n", + ")] = Lat\n" ] }, { "cell_type": "code", "execution_count": null, - "id": "85d74ae8-2d3d-462c-91d1-23da42ef1833", - "metadata": { - "tags": [] - }, + "id": "1845d232-dbc9-48ec-b10a-a6c2a757d2d5", + "metadata": {}, "outputs": [], - "source": [] + "source": [ + "fig, axes = plt.subplots(figsize=(20,8), ncols=2)\n", + "pbar_lon = ds['longitude'].plot(ax=axes[0], cmap=\"Blues\", vmin=-160, vmax=-105, levels=np.arange(-160,-105,3), cbar_kwargs={\"location\": \"top\"})\n", + "pbar_lon.colorbar.ax.tick_params(labelsize=14)\n", + "pbar_lon.colorbar.set_label(r'Longitude ($^\\circ$E)', fontsize=16, weight='bold')\n", + "Lon_masked.plot(ax=axes[0], cmap=\"Greys_r\",vmin=-160,vmax=20, add_colorbar=False, alpha=0.8)\n", + "\n", + "# Optional: Set limits if not automatically handling it\n", + "axes[0].set_xlim([ds['xtrack'].min(),ds['xtrack'].max()])\n", + "axes[0].set_ylim([ds['mirror_step'].min(),ds['mirror_step'].max()])\n", + "\n", + "pbar_lat = ds['latitude'].plot(ax=axes[1], vmin=15, vmax=62.5, levels=20, cmap='Reds',cbar_kwargs={\"location\": \"top\"})\n", + "pbar_lat.colorbar.ax.tick_params(labelsize=14)\n", + "pbar_lat.colorbar.set_label(r'Latitude ($^\\circ$N)', fontsize=16, weight='bold')\n", + "Lat_masked.plot(ax=axes[1], cmap=\"Greys_r\",vmin=40,vmax=90, add_colorbar=False, alpha=0.8)\n", + "\n", + "\n", + "plt.setp(axes[0].get_xticklabels(), fontsize=15)\n", + "plt.setp(axes[0].get_yticklabels(), fontsize=15)\n", + "axes[0].set_xlabel('xtrack', fontsize=17.5)\n", + "axes[0].set_ylabel('mirror_step', fontsize=17.5)\n", + "\n", + "plt.setp(axes[1].get_xticklabels(), fontsize=15)\n", + "plt.setp(axes[1].get_yticklabels(), fontsize=15);\n", + "axes[1].set_xlabel('xtrack', fontsize=17.5)\n", + "axes[1].set_ylabel('mirror_step', fontsize=17.5)\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "id": "2daa6491-1b52-49d1-981b-8af0dd043e02", + "metadata": {}, + "source": [ + "## Now define all variables to download" + ] }, { "cell_type": "code", "execution_count": null, - "id": "81689feb-fa7e-41ab-a479-a794f4c7490d", + "id": "644a6549-5fa3-4a38-8a16-19341e0812f9", "metadata": {}, "outputs": [], - "source": [] + "source": [ + "Vars = dims + [\n", + " \"/product/main_data_quality_flag\",\n", + " \"/product/vertical_column_troposphere\",\n", + " \"/product/vertical_column_stratosphere\",\n", + " \"/geolocation/time\",\n", + " \"/geolocation/longitude\",\n", + " \"/geolocation/latitude\",\n", + " \"/support_data/wind_speed\",\n", + " \"/support_data/terrain_height\",\n", + " \"/support_data/gas_profile\",\n", + " \"/support_data/pbl_height\",\n", + " \"/support_data/temperature_profile\",\n", + "]" + ] + }, + { + "cell_type": "markdown", + "id": "49a742f7-5146-4374-ae0a-b8e29d2d5d08", + "metadata": {}, + "source": [ + "## Download data\n", + "\n", + " At this moment, need to erase any previously downloaded `TEMPO_NO2_L2_*` data \n", + "\n", + " to avoid filename collision\n" + ] }, { "cell_type": "code", "execution_count": null, - "id": "644a6549-5fa3-4a38-8a16-19341e0812f9", + "id": "4f501ecd-1345-43d1-b6eb-1710fe24bdb1", + "metadata": {}, + "outputs": [], + "source": [ + "%%time\n", + "dap_to_netcdf(cmr_urls, session=my_session, \n", + " keep_variables = Vars,\n", + " dim_slices= slices,\n", + " output_path=output_path)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "92af1853-98ce-42fd-b399-344764826142", "metadata": {}, "outputs": [], "source": []