Skip to content

Commit fb26227

Browse files
author
asteiker
committed
updated comments
1 parent 68c238b commit fb26227

File tree

2 files changed

+142
-29
lines changed

2 files changed

+142
-29
lines changed

‎notebooks/ICESat-2_MODIS_Arctic_Sea_Ice/Data access for all datasets.ipynb

Lines changed: 104 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@
99
},
1010
{
1111
"cell_type": "code",
12-
"execution_count": null,
12+
"execution_count": 1,
1313
"metadata": {},
1414
"outputs": [],
1515
"source": [
@@ -38,7 +38,7 @@
3838
},
3939
{
4040
"cell_type": "code",
41-
"execution_count": null,
41+
"execution_count": 2,
4242
"metadata": {},
4343
"outputs": [],
4444
"source": [
@@ -53,7 +53,7 @@
5353
},
5454
{
5555
"cell_type": "code",
56-
"execution_count": null,
56+
"execution_count": 3,
5757
"metadata": {},
5858
"outputs": [],
5959
"source": [
@@ -81,7 +81,7 @@
8181
},
8282
{
8383
"cell_type": "code",
84-
"execution_count": null,
84+
"execution_count": 4,
8585
"metadata": {},
8686
"outputs": [],
8787
"source": [
@@ -95,7 +95,7 @@
9595
},
9696
{
9797
"cell_type": "code",
98-
"execution_count": null,
98+
"execution_count": 5,
9999
"metadata": {
100100
"jupyter": {
101101
"source_hidden": true
@@ -120,9 +120,25 @@
120120
},
121121
{
122122
"cell_type": "code",
123-
"execution_count": null,
123+
"execution_count": 6,
124124
"metadata": {},
125-
"outputs": [],
125+
"outputs": [
126+
{
127+
"name": "stdout",
128+
"output_type": "stream",
129+
"text": [
130+
"There are 2 granules of ATL10 version 005 over my area and time of interest.\n",
131+
"The average size of each granule is 168.34 MB and the total size of all 2 granules is 336.69 MB\n",
132+
"\n",
133+
"There are 4 granules of ATL07 version 005 over my area and time of interest.\n",
134+
"The average size of each granule is 320.07 MB and the total size of all 4 granules is 1280.29 MB\n",
135+
"\n",
136+
"There are 13 granules of MOD29 version 61 over my area and time of interest.\n",
137+
"The average size of each granule is 2.80 MB and the total size of all 13 granules is 36.40 MB\n",
138+
"\n"
139+
]
140+
}
141+
],
126142
"source": [
127143
"# Query number of granules (paging over results)\n",
128144
"granule_search_url = 'https://cmr.earthdata.nasa.gov/search/granules'\n",
@@ -180,13 +196,21 @@
180196
},
181197
{
182198
"cell_type": "code",
183-
"execution_count": null,
199+
"execution_count": 7,
184200
"metadata": {},
185-
"outputs": [],
201+
"outputs": [
202+
{
203+
"name": "stdin",
204+
"output_type": "stream",
205+
"text": [
206+
"Earthdata Login password: ·········\n"
207+
]
208+
}
209+
],
186210
"source": [
187-
"uid = '' # Enter Earthdata Login user name\n",
211+
"uid = 'amy.steiker' # Enter Earthdata Login user name\n",
188212
"pswd = getpass.getpass('Earthdata Login password: ') # Input and store Earthdata Login password\n",
189-
"email = '' # Enter email associated with Earthata Login account"
213+
"email = '[email protected]' # Enter email associated with Earthata Login account"
190214
]
191215
},
192216
{
@@ -198,7 +222,7 @@
198222
},
199223
{
200224
"cell_type": "code",
201-
"execution_count": null,
225+
"execution_count": 8,
202226
"metadata": {},
203227
"outputs": [],
204228
"source": [
@@ -226,7 +250,7 @@
226250
},
227251
{
228252
"cell_type": "code",
229-
"execution_count": null,
253+
"execution_count": 9,
230254
"metadata": {},
231255
"outputs": [],
232256
"source": [
@@ -267,7 +291,7 @@
267291
},
268292
{
269293
"cell_type": "code",
270-
"execution_count": null,
294+
"execution_count": 10,
271295
"metadata": {},
272296
"outputs": [],
273297
"source": [
@@ -302,9 +326,22 @@
302326
},
303327
{
304328
"cell_type": "code",
305-
"execution_count": null,
329+
"execution_count": 11,
306330
"metadata": {},
307-
"outputs": [],
331+
"outputs": [
332+
{
333+
"name": "stdout",
334+
"output_type": "stream",
335+
"text": [
336+
"\n",
337+
"https://n5eil02u.ecs.nsidc.org/egi/request?short_name=ATL10&version=005&bounding_box=140,72,153,80&temporal=2019-03-23T00:00:00Z,2019-03-23T23:59:59Z&bbox=140,72,153,80&time=2019-03-23T00:00:00,2019-03-23T23:59:59&[email protected]&request_mode=async&page_size=2000&page_num=1\n",
338+
"\n",
339+
"https://n5eil02u.ecs.nsidc.org/egi/request?short_name=ATL07&version=005&bounding_box=140,72,153,80&temporal=2019-03-23T00:00:00Z,2019-03-23T23:59:59Z&bbox=140,72,153,80&time=2019-03-23T00:00:00,2019-03-23T23:59:59&coverage=/gt1l/sea_ice_segments/delta_time,/gt1l/sea_ice_segments/latitude,/gt1l/sea_ice_segments/longitude,/gt1l/sea_ice_segments/heights/height_segment_confidence,/gt1l/sea_ice_segments/heights/height_segment_height,/gt1l/sea_ice_segments/heights/height_segment_quality,/gt1l/sea_ice_segments/heights/height_segment_surface_error_est,/gt2l/sea_ice_segments/delta_time,/gt2l/sea_ice_segments/latitude,/gt2l/sea_ice_segments/longitude,/gt2l/sea_ice_segments/heights/height_segment_confidence,/gt2l/sea_ice_segments/heights/height_segment_height,/gt2l/sea_ice_segments/heights/height_segment_quality,/gt2l/sea_ice_segments/heights/height_segment_surface_error_est,/gt3l/sea_ice_segments/delta_time,/gt3l/sea_ice_segments/latitude,/gt3l/sea_ice_segments/longitude,/gt3l/sea_ice_segments/heights/height_segment_confidence,/gt3l/sea_ice_segments/heights/height_segment_height,/gt3l/sea_ice_segments/heights/height_segment_quality,/gt3l/sea_ice_segments/heights/height_segment_surface_error_est&[email protected]&request_mode=async&page_size=2000&page_num=1\n",
340+
"\n",
341+
"https://n5eil02u.ecs.nsidc.org/egi/request?short_name=MOD29&version=61&bounding_box=140,72,153,80&temporal=2019-03-23T00:00:00Z,2019-03-23T23:59:59Z&bbox=140,72,153,80&[email protected]&request_mode=async&page_size=2000&page_num=1\n"
342+
]
343+
}
344+
],
308345
"source": [
309346
"endpoint_list = [] \n",
310347
"for k, v in data_dict.items():\n",
@@ -340,9 +377,55 @@
340377
},
341378
{
342379
"cell_type": "code",
343-
"execution_count": null,
380+
"execution_count": 12,
344381
"metadata": {},
345-
"outputs": [],
382+
"outputs": [
383+
{
384+
"name": "stdout",
385+
"output_type": "stream",
386+
"text": [
387+
"ATL10 Order: 1\n",
388+
"Request HTTP response: 201\n",
389+
"order ID: 5000003709700\n",
390+
"status URL: https://n5eil02u.ecs.nsidc.org/egi/request/5000003709700\n",
391+
"HTTP response from order response URL: 201\n",
392+
"Data request 1 is submitting...\n",
393+
"Initial request status is processing\n",
394+
"Status is not complete. Trying again.\n",
395+
"Retry request status is: complete\n",
396+
"Zip download URL: https://n5eil02u.ecs.nsidc.org/esir/5000003709700.zip\n",
397+
"Beginning download of zipped output...\n",
398+
"Data request 1 is complete.\n",
399+
"\n",
400+
"ATL07 Order: 1\n",
401+
"Request HTTP response: 201\n",
402+
"order ID: 5000003709701\n",
403+
"status URL: https://n5eil02u.ecs.nsidc.org/egi/request/5000003709701\n",
404+
"HTTP response from order response URL: 201\n",
405+
"Data request 1 is submitting...\n",
406+
"Initial request status is processing\n",
407+
"Status is not complete. Trying again.\n",
408+
"Retry request status is: complete\n",
409+
"Zip download URL: https://n5eil02u.ecs.nsidc.org/esir/5000003709701.zip\n",
410+
"Beginning download of zipped output...\n",
411+
"Data request 1 is complete.\n",
412+
"\n",
413+
"MOD29 Order: 1\n",
414+
"Request HTTP response: 201\n",
415+
"order ID: 5000003709702\n",
416+
"status URL: https://n5eil02u.ecs.nsidc.org/egi/request/5000003709702\n",
417+
"HTTP response from order response URL: 201\n",
418+
"Data request 1 is submitting...\n",
419+
"Initial request status is processing\n",
420+
"Status is not complete. Trying again.\n",
421+
"Retry request status is: complete\n",
422+
"Zip download URL: https://n5eil02u.ecs.nsidc.org/esir/5000003709702.zip\n",
423+
"Beginning download of zipped output...\n",
424+
"Data request 1 is complete.\n",
425+
"\n"
426+
]
427+
}
428+
],
346429
"source": [
347430
"from xml.etree import ElementTree as ET\n",
348431
"\n",
@@ -447,7 +530,7 @@
447530
},
448531
{
449532
"cell_type": "code",
450-
"execution_count": null,
533+
"execution_count": 13,
451534
"metadata": {},
452535
"outputs": [],
453536
"source": [
@@ -466,7 +549,7 @@
466549
],
467550
"metadata": {
468551
"kernelspec": {
469-
"display_name": "Python 3",
552+
"display_name": "Python 3 (ipykernel)",
470553
"language": "python",
471554
"name": "python3"
472555
},
@@ -480,7 +563,7 @@
480563
"name": "python",
481564
"nbconvert_exporter": "python",
482565
"pygments_lexer": "ipython3",
483-
"version": "3.7.3"
566+
"version": "3.9.13"
484567
}
485568
},
486569
"nbformat": 4,

‎notebooks/ICESat-2_MODIS_Arctic_Sea_Ice/Visualize and Analyze Data.ipynb

Lines changed: 38 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@
3030
},
3131
{
3232
"cell_type": "code",
33-
"execution_count": null,
33+
"execution_count": 1,
3434
"metadata": {},
3535
"outputs": [],
3636
"source": [
@@ -57,6 +57,8 @@
5757
"cell_type": "markdown",
5858
"metadata": {},
5959
"source": [
60+
"TODO: Need to make some updates to Customize and Access notebook so it's not so confusing to have to run the other \"all datasets\" notebook\n",
61+
"\n",
6062
"During the live tutorial in 2019, we temporarily staged data on Amazon Web Services (AWS). Those commands are copied below for reference. Alternatively, we can access all of the granules needed for this tutorial using the `Data access for all datasets` notebook located in this folder. \n",
6163
"\n",
6264
"```bash\n",
@@ -102,7 +104,7 @@
102104
},
103105
{
104106
"cell_type": "code",
105-
"execution_count": null,
107+
"execution_count": 3,
106108
"metadata": {},
107109
"outputs": [],
108110
"source": [
@@ -115,23 +117,23 @@
115117
" '/gt1l/sea_ice_segments/heights/height_segment_height',\n",
116118
" '/gt1l/sea_ice_segments/heights/height_segment_quality',\n",
117119
" '/gt1l/sea_ice_segments/heights/height_segment_surface_error_est',\n",
118-
"# '/gt1l/sea_ice_segments/heights/height_segment_length_seg',\n",
120+
" '/gt1l/sea_ice_segments/heights/height_segment_length_seg',\n",
119121
" '/gt2l/sea_ice_segments/delta_time',\n",
120122
" '/gt2l/sea_ice_segments/latitude',\n",
121123
" '/gt2l/sea_ice_segments/longitude',\n",
122124
" '/gt2l/sea_ice_segments/heights/height_segment_confidence',\n",
123125
" '/gt2l/sea_ice_segments/heights/height_segment_height',\n",
124126
" '/gt2l/sea_ice_segments/heights/height_segment_quality',\n",
125127
" '/gt2l/sea_ice_segments/heights/height_segment_surface_error_est',\n",
126-
"# '/gt2l/sea_ice_segments/heights/height_segment_length_seg',\n",
128+
" '/gt2l/sea_ice_segments/heights/height_segment_length_seg',\n",
127129
" '/gt3l/sea_ice_segments/delta_time',\n",
128130
" '/gt3l/sea_ice_segments/latitude',\n",
129131
" '/gt3l/sea_ice_segments/longitude',\n",
130132
" '/gt3l/sea_ice_segments/heights/height_segment_confidence',\n",
131133
" '/gt3l/sea_ice_segments/heights/height_segment_height',\n",
132134
" '/gt3l/sea_ice_segments/heights/height_segment_quality',\n",
133-
" '/gt3l/sea_ice_segments/heights/height_segment_surface_error_est'],\n",
134-
"# '/gt3l/sea_ice_segments/heights/height_segment_length_seg'],\n",
135+
" '/gt3l/sea_ice_segments/heights/height_segment_surface_error_est',\n",
136+
" '/gt3l/sea_ice_segments/heights/height_segment_length_seg'],\n",
135137
" 'ATL10': [\n",
136138
" '/gt1l/freeboard_beam_segment/beam_freeboard/delta_time',\n",
137139
" '/gt1l/freeboard_beam_segment/beam_freeboard/latitude',\n",
@@ -181,14 +183,42 @@
181183
},
182184
{
183185
"cell_type": "code",
184-
"execution_count": null,
186+
"execution_count": 4,
185187
"metadata": {
186188
"tags": []
187189
},
188-
"outputs": [],
190+
"outputs": [
191+
{
192+
"name": "stdout",
193+
"output_type": "stream",
194+
"text": [
195+
"Variable /gt2l/sea_ice_segments/heights/height_segment_length_seg not found in ./Outputs/processed_ATL07-01_20190323101341_12980201_005_01.h5. Likely an empty granule.\n"
196+
]
197+
},
198+
{
199+
"ename": "KeyError",
200+
"evalue": "\"Unable to open object (object 'height_segment_length_seg' doesn't exist)\"",
201+
"output_type": "error",
202+
"traceback": [
203+
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
204+
"\u001b[0;31mKeyError\u001b[0m Traceback (most recent call last)",
205+
"Input \u001b[0;32mIn [4]\u001b[0m, in \u001b[0;36m<cell line: 5>\u001b[0;34m()\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[38;5;66;03m# Updated based on V005 data\u001b[39;00m\n\u001b[1;32m 2\u001b[0m \u001b[38;5;66;03m#TODO: Grab file names based on persistent data acquisition date to mitigate new version file name updates\u001b[39;00m\n\u001b[1;32m 3\u001b[0m \n\u001b[1;32m 4\u001b[0m \u001b[38;5;66;03m# Load ATL07 files\u001b[39;00m\n\u001b[0;32m----> 5\u001b[0m atl07_1298 \u001b[38;5;241m=\u001b[39m \u001b[43mfn\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mload_icesat2_as_dataframe\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43m./Outputs/processed_ATL07-01_20190323101341_12980201_005_01.h5\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mVARIABLES\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 6\u001b[0m atl07_1305 \u001b[38;5;241m=\u001b[39m fn\u001b[38;5;241m.\u001b[39mload_icesat2_as_dataframe(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m./Outputs/processed_ATL07-01_20190323211343_13050201_005_01.h5\u001b[39m\u001b[38;5;124m'\u001b[39m, VARIABLES)\n\u001b[1;32m 7\u001b[0m atl07 \u001b[38;5;241m=\u001b[39m pd\u001b[38;5;241m.\u001b[39mconcat([atl07_1298,atl07_1305]) \u001b[38;5;66;03m#concatenate dataframes to use in plot below\u001b[39;00m\n",
206+
"File \u001b[0;32m~/other-repos/NSIDC-Data-Tutorials/notebooks/ICESat-2_MODIS_Arctic_Sea_Ice/tutorial_helper_functions.py:308\u001b[0m, in \u001b[0;36mload_icesat2_as_dataframe\u001b[0;34m(filepath, VARIABLES)\u001b[0m\n\u001b[1;32m 306\u001b[0m variable \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mstr\u001b[39m(variable)\n\u001b[1;32m 307\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 308\u001b[0m values \u001b[38;5;241m=\u001b[39m \u001b[43mds\u001b[49m\u001b[43m[\u001b[49m\u001b[43mvariable\u001b[49m\u001b[43m]\u001b[49m[:]\n\u001b[1;32m 309\u001b[0m \u001b[38;5;66;03m# Convert invalid data to np.nan (only for float columns)\u001b[39;00m\n\u001b[1;32m 310\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mfloat\u001b[39m\u001b[38;5;124m'\u001b[39m \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mstr\u001b[39m(values\u001b[38;5;241m.\u001b[39mdtype):\n",
207+
"File \u001b[0;32mh5py/_objects.pyx:54\u001b[0m, in \u001b[0;36mh5py._objects.with_phil.wrapper\u001b[0;34m()\u001b[0m\n",
208+
"File \u001b[0;32mh5py/_objects.pyx:55\u001b[0m, in \u001b[0;36mh5py._objects.with_phil.wrapper\u001b[0;34m()\u001b[0m\n",
209+
"File \u001b[0;32m/srv/conda/envs/notebook/lib/python3.9/site-packages/h5py/_hl/group.py:328\u001b[0m, in \u001b[0;36mGroup.__getitem__\u001b[0;34m(self, name)\u001b[0m\n\u001b[1;32m 326\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mInvalid HDF5 object reference\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m 327\u001b[0m \u001b[38;5;28;01melif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(name, (\u001b[38;5;28mbytes\u001b[39m, \u001b[38;5;28mstr\u001b[39m)):\n\u001b[0;32m--> 328\u001b[0m oid \u001b[38;5;241m=\u001b[39m \u001b[43mh5o\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mopen\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mid\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_e\u001b[49m\u001b[43m(\u001b[49m\u001b[43mname\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mlapl\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_lapl\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 329\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m 330\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mTypeError\u001b[39;00m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mAccessing a group is done with bytes or str, \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 331\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m not \u001b[39m\u001b[38;5;132;01m{}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;241m.\u001b[39mformat(\u001b[38;5;28mtype\u001b[39m(name)))\n",
210+
"File \u001b[0;32mh5py/_objects.pyx:54\u001b[0m, in \u001b[0;36mh5py._objects.with_phil.wrapper\u001b[0;34m()\u001b[0m\n",
211+
"File \u001b[0;32mh5py/_objects.pyx:55\u001b[0m, in \u001b[0;36mh5py._objects.with_phil.wrapper\u001b[0;34m()\u001b[0m\n",
212+
"File \u001b[0;32mh5py/h5o.pyx:190\u001b[0m, in \u001b[0;36mh5py.h5o.open\u001b[0;34m()\u001b[0m\n",
213+
"\u001b[0;31mKeyError\u001b[0m: \"Unable to open object (object 'height_segment_length_seg' doesn't exist)\""
214+
]
215+
}
216+
],
189217
"source": [
190218
"# Updated based on V005 data\n",
191219
"#TODO: Grab file names based on persistent data acquisition date to mitigate new version file name updates\n",
220+
"#Future TODO: Use icepyx as reader instead of Bruce's function\n",
221+
"\n",
192222
"\n",
193223
"# Load ATL07 files\n",
194224
"atl07_1298 = fn.load_icesat2_as_dataframe('./Outputs/processed_ATL07-01_20190323101341_12980201_005_01.h5', VARIABLES)\n",

0 commit comments

Comments
 (0)