From eee2dfe4384a3d29d9e439b8c2e14d6f23d7b371 Mon Sep 17 00:00:00 2001 From: Jessica Scheick Date: Tue, 7 Jun 2022 14:53:44 -0400 Subject: [PATCH 01/17] add framed notebooks for new scipy2022 tutorial content --- _toc.yml | 9 ++ workshops/scipy2022/cleaning_real_data.ipynb | 44 ++++++++ workshops/scipy2022/xarray_ecosystem.ipynb | 105 +++++++++++++++++++ 3 files changed, 158 insertions(+) create mode 100644 workshops/scipy2022/cleaning_real_data.ipynb create mode 100644 workshops/scipy2022/xarray_ecosystem.ipynb diff --git a/_toc.yml b/_toc.yml index baa3b968..fd9907c9 100644 --- a/_toc.yml +++ b/_toc.yml @@ -26,6 +26,15 @@ parts: - file: workshops/online-tutorial-series/02_indexing - file: workshops/online-tutorial-series/03_computation - file: workshops/scipy2022/README + sections: + - url: https://tutorial.xarray.dev/scipy-tutorial/00_overview + - url: https://tutorial.xarray.dev/scipy-tutorial/01_datastructures_and_io + - file: workshops/scipy2022/cleaning_real_data + - url: https://tutorial.xarray.dev/scipy-tutorial/02_working_with_labeled_data + - url: https://tutorial.xarray.dev/scipy-tutorial/03_computation_with_xarray + - url: https://tutorial.xarray.dev/scipy-tutorial/04_plotting_and_visualization.html + - file: workshops/scipy2022/xarray_ecosystem + - url: https://tutorial.xarray.dev/scipy-tutorial/06_xarray_and_dask - caption: Reference chapters: diff --git a/workshops/scipy2022/cleaning_real_data.ipynb b/workshops/scipy2022/cleaning_real_data.ipynb new file mode 100644 index 00000000..f05463b7 --- /dev/null +++ b/workshops/scipy2022/cleaning_real_data.ipynb @@ -0,0 +1,44 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Cleaning Real Data\n", + "\n", + "Real-world datasets are often imperfect. This can pose a challenge for new and experienced users alike working with software that expects clean, uniform data. Here we showcase how to troubleshoot some common challenges encountered working with real data." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import xarray as xr\n", + "# Your code here" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.12" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/workshops/scipy2022/xarray_ecosystem.ipynb b/workshops/scipy2022/xarray_ecosystem.ipynb new file mode 100644 index 00000000..bff585c6 --- /dev/null +++ b/workshops/scipy2022/xarray_ecosystem.ipynb @@ -0,0 +1,105 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "\n", + "# Xarray Ecosystem\n", + "\n", + "Xarray is easily extensible.\n", + "This means it is easy to add onto to build custom packages that tackle particular computational problems.\n", + "Here we introduce two popular and widely used extensions that are installable as their own packages (via conda and pip).\n", + "\n", + "- [rioxarray](https://corteva.github.io/rioxarray/stable/index.html), for working with geospatial raster data using rasterio\n", + "- [pint-xarray](https://pint-xarray.readthedocs.io/en/latest/), for unit-aware computations using pint.\n", + "\n", + "## Rioxarray\n", + "more details about rioxarray here\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import xarray as xr\n", + "import rioxarray # this activates the rio accessor" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "jupyter": { + "source_hidden": true + } + }, + "outputs": [], + "source": [ + "# rioxarray eample" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Pint\n", + "\n", + "more details about pint here\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# to be able to read unit attributes following the CF conventions\n", + "import cf_xarray.units # must be imported before pint_xarray\n", + "import pint_xarray\n", + "\n", + "xr.set_options(display_expand_data=False)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# pint example" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.12" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} From 81e9127cf8d6718e8ce669b4c9e96bc7636c832e Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 7 Jun 2022 18:56:53 +0000 Subject: [PATCH 02/17] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- workshops/scipy2022/cleaning_real_data.ipynb | 1 + workshops/scipy2022/xarray_ecosystem.ipynb | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/workshops/scipy2022/cleaning_real_data.ipynb b/workshops/scipy2022/cleaning_real_data.ipynb index f05463b7..47552fbb 100644 --- a/workshops/scipy2022/cleaning_real_data.ipynb +++ b/workshops/scipy2022/cleaning_real_data.ipynb @@ -16,6 +16,7 @@ "outputs": [], "source": [ "import xarray as xr\n", + "\n", "# Your code here" ] } diff --git a/workshops/scipy2022/xarray_ecosystem.ipynb b/workshops/scipy2022/xarray_ecosystem.ipynb index bff585c6..ac57e933 100644 --- a/workshops/scipy2022/xarray_ecosystem.ipynb +++ b/workshops/scipy2022/xarray_ecosystem.ipynb @@ -26,7 +26,7 @@ "outputs": [], "source": [ "import xarray as xr\n", - "import rioxarray # this activates the rio accessor" + "import rioxarray # this activates the rio accessor" ] }, { From 43eee64ad759547ab5b330eb0a716bc94093232c Mon Sep 17 00:00:00 2001 From: Jessica Scheick Date: Fri, 10 Jun 2022 13:03:06 -0400 Subject: [PATCH 03/17] separate tutorial content into smaller notebooks --- _toc.yml | 14 +- ...b => 02.1_working_with_labeled_data.ipynb} | 184 -------- .../02.2_manipulating_dimensions.ipynb | 129 ++++++ .../02.3_aligning_data_objects.ipynb | 163 +++++++ .../03.1_computation_with_xarray.ipynb | 418 ++++++++++++++++++ ...y.ipynb => 03.2_groupby_with_xarray.ipynb} | 324 +------------- 6 files changed, 723 insertions(+), 509 deletions(-) rename scipy-tutorial/{02_working_with_labeled_data.ipynb => 02.1_working_with_labeled_data.ipynb} (62%) create mode 100644 scipy-tutorial/02.2_manipulating_dimensions.ipynb create mode 100644 scipy-tutorial/02.3_aligning_data_objects.ipynb create mode 100644 scipy-tutorial/03.1_computation_with_xarray.ipynb rename scipy-tutorial/{03_computation_with_xarray.ipynb => 03.2_groupby_with_xarray.ipynb} (64%) diff --git a/_toc.yml b/_toc.yml index fd9907c9..9812c705 100644 --- a/_toc.yml +++ b/_toc.yml @@ -6,8 +6,11 @@ parts: chapters: - file: scipy-tutorial/00_overview - file: scipy-tutorial/01_datastructures_and_io - - file: scipy-tutorial/02_working_with_labeled_data - - file: scipy-tutorial/03_computation_with_xarray + - file: scipy-tutorial/02.1_working_with_labeled_data + - file: scipy-tutorial/02.2_manipulating_dimensions + - file: scipy-tutorial/02.3_aligning_data_objects + - file: scipy-tutorial/03.1_computation_with_xarray + - file: scipy-tutorial/03.2_groupby_with_xarray - file: scipy-tutorial/04_plotting_and_visualization - file: scipy-tutorial/05_intro_to_dask - file: scipy-tutorial/06_xarray_and_dask @@ -30,8 +33,11 @@ parts: - url: https://tutorial.xarray.dev/scipy-tutorial/00_overview - url: https://tutorial.xarray.dev/scipy-tutorial/01_datastructures_and_io - file: workshops/scipy2022/cleaning_real_data - - url: https://tutorial.xarray.dev/scipy-tutorial/02_working_with_labeled_data - - url: https://tutorial.xarray.dev/scipy-tutorial/03_computation_with_xarray + - url: https://tutorial.xarray.dev/scipy-tutorial/02.1_working_with_labeled_data + - url: https://tutorial.xarray.dev/scipy-tutorial/02.2_manipulating_dimensions + - url: https://tutorial.xarray.dev/scipy-tutorial/02.3_aligning_data_objects + - url: https://tutorial.xarray.dev/scipy-tutorial/03.1_computation_with_xarray + - url: https://tutorial.xarray.dev/scipy-tutorial/03.2_groupby_with_xarray - url: https://tutorial.xarray.dev/scipy-tutorial/04_plotting_and_visualization.html - file: workshops/scipy2022/xarray_ecosystem - url: https://tutorial.xarray.dev/scipy-tutorial/06_xarray_and_dask diff --git a/scipy-tutorial/02_working_with_labeled_data.ipynb b/scipy-tutorial/02.1_working_with_labeled_data.ipynb similarity index 62% rename from scipy-tutorial/02_working_with_labeled_data.ipynb rename to scipy-tutorial/02.1_working_with_labeled_data.ipynb index 0d74c4ef..f3c6b32d 100644 --- a/scipy-tutorial/02_working_with_labeled_data.ipynb +++ b/scipy-tutorial/02.1_working_with_labeled_data.ipynb @@ -301,190 +301,6 @@ "source": [ "# your code here" ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Interpolation\n", - "\n", - "If we want to look at values between the current grid cells (interpolation), we\n", - "can do that with `interp` (requires `scipy`):\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "arr.interp(\n", - " x=np.linspace(2, 6, 10),\n", - " y=pd.date_range(\"2009-04-01\", \"2009-04-30\", freq=\"D\"),\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "when trying to extrapolate, the resulting values will be `nan`.\n", - "\n", - "If we already have a object with the desired coordinates, we can use\n", - "`interp_like`:\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "other = xr.DataArray(\n", - " dims=(\"x\", \"y\"),\n", - " coords={\n", - " \"x\": np.linspace(2, 4, 10),\n", - " \"y\": pd.date_range(\"2009-04-01\", \"2009-04-30\", freq=\"D\"),\n", - " },\n", - ")\n", - "arr.interp_like(other)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Exercises\n", - "\n", - "Increase the step size along latitude and longitude from 2.5 degrees to 1\n", - "degree.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# your code here" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Broadcasting and automatic alignment\n", - "\n", - "Labels help with combining arrays with different coordinates:\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "a = xr.DataArray(\n", - " np.random.randn(3, 4),\n", - " dims=(\"x\", \"y\"),\n", - " coords={\"x\": [\"a\", \"b\", \"c\"], \"y\": np.arange(4)},\n", - ")\n", - "b = xr.DataArray(\n", - " np.random.randn(2, 7),\n", - " dims=(\"x\", \"y\"),\n", - " coords={\"x\": [\"b\", \"d\"], \"y\": [-2, -1, 0, 1, 2, 3, 4]},\n", - ")\n", - "\n", - "a + b" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "This will automatically select only common labels from both arrays (a inner\n", - "join) and then perform the operation.\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Broadcasting works similar:\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "arr1 = xr.DataArray(\n", - " np.random.randn(3),\n", - " dims=\"x\",\n", - " coords={\"x\": [\"a\", \"b\", \"c\"]},\n", - ")\n", - "arr2 = xr.DataArray(\n", - " np.random.randn(4),\n", - " dims=\"y\",\n", - " coords={\"y\": np.arange(4)},\n", - ")\n", - "\n", - "arr1 + arr2" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "where both arrays were automatically broadcasted against each other:\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "arr1_, arr2_ = xr.broadcast(arr1, arr2)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "arr1_" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "arr2_" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "and then the operation (a sum) was executed.\n", - "\n", - "We can also call `align` speciically with different options.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "a_al, b_al = xr.align(a, b, join=\"inner\")\n", - "b_al" - ] } ], "metadata": { diff --git a/scipy-tutorial/02.2_manipulating_dimensions.ipynb b/scipy-tutorial/02.2_manipulating_dimensions.ipynb new file mode 100644 index 00000000..d24f69c9 --- /dev/null +++ b/scipy-tutorial/02.2_manipulating_dimensions.ipynb @@ -0,0 +1,129 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "\n", + "# Manipulating Dimensions (Data Resolution)\n", + "\n", + "Sometimes we need to change the resolution of our data. We might need to look at inferred values between dimension (grid) spaces\n", + "or change the dimension spacing completely (for instance to add another variable). Learning goals:\n", + "\n", + "- Interpolate data to new coordinates" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import numpy as np\n", + "import pandas as pd\n", + "import xarray as xr\n", + "\n", + "np.random.seed(0)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "arr = xr.DataArray(np.random.randn(3, 4), dims=(\"x\", \"y\"))\n", + "arr" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Interpolation\n", + "\n", + "If we want to look at values between the current grid cells (interpolation), we\n", + "can do that with `interp` (requires `scipy`):\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "arr.interp(\n", + " x=np.linspace(2, 6, 10),\n", + " y=pd.date_range(\"2009-04-01\", \"2009-04-30\", freq=\"D\"),\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "when trying to extrapolate, the resulting values will be `nan`.\n", + "\n", + "If we already have a object with the desired coordinates, we can use\n", + "`interp_like`:\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "other = xr.DataArray(\n", + " dims=(\"x\", \"y\"),\n", + " coords={\n", + " \"x\": np.linspace(2, 4, 10),\n", + " \"y\": pd.date_range(\"2009-04-01\", \"2009-04-30\", freq=\"D\"),\n", + " },\n", + ")\n", + "arr.interp_like(other)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Exercises\n", + "\n", + "Increase the step size along latitude and longitude from 2.5 degrees to 1\n", + "degree.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# your code here" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.12" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/scipy-tutorial/02.3_aligning_data_objects.ipynb b/scipy-tutorial/02.3_aligning_data_objects.ipynb new file mode 100644 index 00000000..55f6c5f6 --- /dev/null +++ b/scipy-tutorial/02.3_aligning_data_objects.ipynb @@ -0,0 +1,163 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "\n", + "# Working with Multiple Datasets\n", + "\n", + "Learning goals:\n", + "\n", + "- Perform operations across multiple datasets" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import numpy as np\n", + "import xarray as xr\n", + "\n", + "np.random.seed(0)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Broadcasting and automatic alignment\n", + "\n", + "Broadcasting refers to the alignment of arrays with different numbers of\n", + "dimensions. Numpy's broadcasting rules, based on array shape, can sometimes be\n", + "difficult to understand and remember. Xarray does broadcasting by dimension name,\n", + "rather than array shape. This is a huge convenience." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "arr1 = xr.DataArray(\n", + " np.random.randn(3, 4),\n", + " dims=(\"x\", \"y\"),\n", + " coords={\"x\": [\"a\", \"b\", \"c\"], \"y\": np.arange(4)},\n", + ")\n", + "arr2 = xr.DataArray(\n", + " np.random.randn(2, 7),\n", + " dims=(\"x\", \"y\"),\n", + " coords={\"x\": [\"b\", \"d\"], \"y\": [-2, -1, 0, 1, 2, 3, 4]},\n", + ")\n", + "\n", + "arr1 + arr2" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "This will automatically select only common labels from both arrays (a inner\n", + "join) and then perform the operation.\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "
\n", + " Warning: If the arrays being broadcasted share a dimension name, but have different coordinates,\n", + " they will first be aligned using Xarray's default align settings (including filling missing values with NaNs).\n", + " If that's not what you want, it's best to call align explicitly before broadcasting.\n", + "
\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### A More Detailed Look\n", + "\n", + "What Xarray is doing automatically to perform the addition operation is a combination of broadcast and align.\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "First, both arrays were automatically broadcasted against each other:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "arr1_, arr2_ = xr.broadcast(arr1, arr2)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "arr1_" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "arr2_" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "and then the operation (a sum) was executed.\n", + "\n", + "We can also call `align` speciically with different options.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "arr1_al, arr2_al = xr.align(arr1, arr2, join=\"inner\")\n", + "arr2_al" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.12" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/scipy-tutorial/03.1_computation_with_xarray.ipynb b/scipy-tutorial/03.1_computation_with_xarray.ipynb new file mode 100644 index 00000000..7baf4074 --- /dev/null +++ b/scipy-tutorial/03.1_computation_with_xarray.ipynb @@ -0,0 +1,418 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "\n", + "# Computation with Xarray\n", + "\n", + "In this lesson, we discuss how to do scientific computations with xarray\n", + "objects. Our learning goals are as follows. By the end of the lesson, we will be\n", + "able to:\n", + "\n", + "- Apply basic arithmetic and numpy functions to xarray DataArrays / Dataset.\n", + "- Use Xarray's label-aware reduction operations (e.g. `mean`, `sum`) weighted\n", + " reductions.\n", + "- Apply arbitrary functions to Xarray data via `apply_ufunc`.\n", + "- Use Xarray's broadcasting to compute on arrays of different dimensionality." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import numpy as np\n", + "import xarray as xr\n", + "from matplotlib import pyplot as plt" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Example Dataset\n", + "\n", + "First we load a dataset. We will use the\n", + "[NOAA Extended Reconstructed Sea Surface Temperature (ERSST) v5](https://www.ncdc.noaa.gov/data-access/marineocean-data/extended-reconstructed-sea-surface-temperature-ersst-v5)\n", + "product, a widely used and trusted gridded compilation of of historical data\n", + "going back to 1854.\n", + "\n", + "Since the data is provided via an\n", + "[OPeNDAP](https://en.wikipedia.org/wiki/OPeNDAP) server, we can load it directly\n", + "without downloading anything:\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "### NOTE: If hundreds of people connect to this server at once and download the same dataset,\n", + "### things might not go so well! Recommended to use the Google Cloud copy instead.\n", + "\n", + "# url = \"http://www.esrl.noaa.gov/psd/thredds/dodsC/Datasets/noaa.ersst.v5/sst.mnmean.nc\"\n", + "# # drop an unnecessary variable which complicates some operations\n", + "# ds = xr.open_dataset(url, drop_variables=[\"time_bnds\"])\n", + "# # will take a minute or two to complete\n", + "# ds = ds.sel(time=slice(\"1960\", \"2018\")).load()\n", + "# ds" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import gcsfs\n", + "\n", + "fs = gcsfs.GCSFileSystem(token=\"anon\")\n", + "ds = xr.open_zarr(\n", + " fs.get_mapper(\"gs://pangeo-noaa-ncei/noaa.ersst.v5.zarr\"), consolidated=True\n", + ").load()\n", + "ds" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Let's do some basic visualizations of the data, just to make sure it looks\n", + "reasonable.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "ds.sst[0].plot(vmin=-2, vmax=30)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Basic Arithmetic\n", + "\n", + "Xarray dataarrays and datasets work seamlessly with arithmetic operators and\n", + "numpy array functions.\n", + "\n", + "For example, imagine we want to convert the temperature (given in Celsius) to\n", + "Kelvin:\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "sst_kelvin = ds.sst + 273.15\n", + "sst_kelvin" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The dimensions and coordinates were preserved following the operation.\n", + "\n", + "
\n", + " Warning: Although many xarray datasets have a units attribute, which is used in plotting,\n", + " Xarray does not inherently understand units. However, work is underway to integrate xarray\n", + " with pint, which will provide full unit-aware operations.\n", + "
\n", + "\n", + "We can apply more complex functions, including numpy ufuncs, to Xarray objects.\n", + "Imagine we wanted to compute the following expression as a function of SST\n", + "($\\Theta$) in Kelvin:\n", + "\n", + "$$ f(\\Theta) = 0.5 \\ln(\\Theta^2) $$\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "f = 0.5 * np.log(sst_kelvin**2)\n", + "f" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Applying Arbitrary Functions\n", + "\n", + "It's awesome that we can call `np.log(ds)` and have it \"just work\". However, not\n", + "all third party libraries work this way.\n", + "\n", + "In this example, we will use functions from the\n", + "[Gibbs Seawater Toolkit](https://teos-10.github.io/GSW-Python/), a package for\n", + "the thermodynamics of seawater. This package provides ufuncs that operate on\n", + "numpy arrays.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import gsw\n", + "\n", + "# an example function\n", + "# http://www.teos-10.org/pubs/gsw/html/gsw_t90_from_t68.html\n", + "?gsw.t90_from_t68" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "gsw.t90_from_t68(ds.sst) # -> returns a numpy array" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "It would be nice to keep our dimensions and coordinates. We can accomplish this\n", + "with `xr.apply_ufunc`.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "xr.apply_ufunc(gsw.t90_from_t68, ds.sst)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "
\n", + " Note: apply_ufunc is a powerful and mysterious function.\n", + " It has many options for doing more complicated things.\n", + " Unfortunately, we don't have time to go into more depth here.\n", + " Please consult the [Xarray docs](http://xarray.pydata.org/en/latest/generated/xarray.apply_ufunc.html) for more details.\n", + "
\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Reductions\n", + "\n", + "Just like in numpy, we can reduce xarray DataArrays along any number of axes:\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "sst = ds.sst\n", + "sst.mean(axis=0)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "sst.mean(axis=(1, 2))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "sst.mean()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "However, rather than performing reductions on axes (as in numpy), we can perform\n", + "them on dimensions. This turns out to be a huge convenience, particularly in\n", + "complex calculations when you can't easily remember which axis corresponds to\n", + "which dimension:\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "sst.mean(dim=\"time\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "All of the standard numpy reductions (e.g. `min`, `max`, `sum`, `std`, etc.) are\n", + "available\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "tags": [] + }, + "source": [ + "### Exercise\n", + "\n", + "Take the mean of `sst` in both longitude and latitude. Make a simple timeseries\n", + "plot:\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# your code here" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Weighted Reductions\n", + "\n", + "For this example, we will create a “weights” array proportional to cosine of latitude.\n", + "Modulo a normalization, this is the correct area-weighting factor for data on a regular lat-lon grid.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "weights = np.cos(np.deg2rad(ds.lat))\n", + "weights.dims" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Thanks to the automatic broadcasting and alignment discussed earlier, if we multiply this by SST, it “just works,” and the arrays are broadcasted properly:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "(ds.sst * weights).dims" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We could imagine computing the weighted spatial mean of SST manually." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "sst_mean = (ds.sst * weights).sum(dim=(\"lon\", \"lat\")) / weights.sum(dim=\"lat\")\n", + "sst_mean.plot()\n", + "plt.title(\"This is wrong!\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "That would be wrong, however, because the denominator (`weights.sum(dim='lat')`)\n", + "needs to be expanded to include the `lon` dimension and modified to account for\n", + "the missing values (land points).\n", + "\n", + "In general, weighted reductions on multidimensional arrays are complicated. To\n", + "make it a bit easier, Xarray provides a mechanism for weighted reductions. It\n", + "does this by creating a special intermediate `DataArrayWeighted` object, to\n", + "which different reduction operations can applied.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "sst_weighted = ds.sst.weighted(weights)\n", + "sst_weighted" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "sst_weighted.mean(dim=(\"lon\", \"lat\")).plot()\n", + "plt.title(\"Correct Global Mean SST\")" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.12" + }, + "widgets": { + "application/vnd.jupyter.widget-state+json": { + "state": {}, + "version_major": 2, + "version_minor": 0 + } + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/scipy-tutorial/03_computation_with_xarray.ipynb b/scipy-tutorial/03.2_groupby_with_xarray.ipynb similarity index 64% rename from scipy-tutorial/03_computation_with_xarray.ipynb rename to scipy-tutorial/03.2_groupby_with_xarray.ipynb index 69ceb920..3373c37e 100644 --- a/scipy-tutorial/03_computation_with_xarray.ipynb +++ b/scipy-tutorial/03.2_groupby_with_xarray.ipynb @@ -6,17 +6,11 @@ "source": [ "\n", "\n", - "# Computation with Xarray\n", + "# Grouped Computations with Xarray\n", "\n", - "In this lesson, we discuss how to do scientific computations with xarray\n", - "objects. Our learning goals are as follows. By the end of the lesson, we will be\n", - "able to:\n", + "In this lesson, we discuss how to do scientific computations with defined \"groups\" of data\n", + "within our xarray objects. Our learning goals are as follows:\n", "\n", - "- Apply basic arithmetic and numpy functions to xarray DataArrays / Dataset.\n", - "- Use Xarray's label-aware reduction operations (e.g. `mean`, `sum`) weighted\n", - " reductions.\n", - "- Apply arbitrary functions to Xarray data via `apply_ufunc`.\n", - "- Use Xarray's broadcasting to compute on arrays of different dimensionality.\n", "- Perform \"split / apply / combine\" workflows in Xarray using `groupby`,\n", " including\n", " - reductions within groups\n", @@ -100,318 +94,6 @@ "ds.sst[0].plot(vmin=-2, vmax=30)" ] }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Basic Arithmetic\n", - "\n", - "Xarray dataarrays and datasets work seamlessly with arithmetic operators and\n", - "numpy array functions.\n", - "\n", - "For example, imagine we want to convert the temperature (given in Celsius) to\n", - "Kelvin:\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "sst_kelvin = ds.sst + 273.15\n", - "sst_kelvin" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The dimensions and coordinates were preserved following the operation.\n", - "\n", - "
\n", - " Warning: Although many xarray datasets have a units attribute, which is used in plotting,\n", - " Xarray does not inherently understand units. However, work is underway to integrate xarray\n", - " with pint, which will provide full unit-aware operations.\n", - "
\n", - "\n", - "We can apply more complex functions, including numpy ufuncs, to Xarray objects.\n", - "Imagine we wanted to compute the following expression as a function of SST\n", - "($\\Theta$) in Kelvin:\n", - "\n", - "$$ f(\\Theta) = 0.5 \\ln(\\Theta^2) $$\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "f = 0.5 * np.log(sst_kelvin**2)\n", - "f" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Applying Arbitrary Functions\n", - "\n", - "It's awesome that we can call `np.log(ds)` and have it \"just work\". However, not\n", - "all third party libraries work this way.\n", - "\n", - "In this example, we will use functions from the\n", - "[Gibbs Seawater Toolkit](https://teos-10.github.io/GSW-Python/), a package for\n", - "the thermodynamics of seawater. This package provides ufuncs that operate on\n", - "numpy arrays.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import gsw\n", - "\n", - "# an example function\n", - "# http://www.teos-10.org/pubs/gsw/html/gsw_t90_from_t68.html\n", - "?gsw.t90_from_t68" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "gsw.t90_from_t68(ds.sst) # -> returns a numpy array" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "It would be nice to keep our dimensions and coordinates. We can accomplish this\n", - "with `xr.apply_ufunc`.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "xr.apply_ufunc(gsw.t90_from_t68, ds.sst)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "
\n", - " Note: apply_ufunc is a powerful and mysterious function.\n", - " It has many options for doing more complicated things.\n", - " Unfortunately, we don't have time to go into more depth here.\n", - " Please consult the [Xarray docs](http://xarray.pydata.org/en/latest/generated/xarray.apply_ufunc.html) for more details.\n", - "
\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Reductions\n", - "\n", - "Just like in numpy, we can reduce xarray DataArrays along any number of axes:\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "sst = ds.sst\n", - "sst.mean(axis=0)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "sst.mean(axis=(1, 2))" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "sst.mean()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "However, rather than performing reductions on axes (as in numpy), we can perform\n", - "them on dimensions. This turns out to be a huge convenience, particularly in\n", - "complex calculations when you can't easily remember which axis corresponds to\n", - "which dimension:\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "sst.mean(dim=\"time\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "All of the standard numpy reductions (e.g. `min`, `max`, `sum`, `std`, etc.) are\n", - "available\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "tags": [] - }, - "source": [ - "### Exercise\n", - "\n", - "Take the mean of `sst` in both longitude and latitude. Make a simple timeseries\n", - "plot:\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# your code here" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Broadcasting\n", - "\n", - "Broadcasting refers to the alignmed of arrays with different numbers of\n", - "dimensions. Numpy's broadcasting rules, based on array shape, can sometimes be\n", - "difficult to understand and remember. Xarray does brodcasting by dimension name,\n", - "rather than array shape. This is a huge convenience.\n", - "\n", - "Let's now create two arrays with some dimensions in common. For this example, we\n", - "will create a \"weights\" array proportional to cosine of latitude. Modulo a\n", - "normalization, this is the correct area-weighting factor for data on a regular\n", - "lat-lon grid.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "weights = np.cos(np.deg2rad(ds.lat))\n", - "weights.dims" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "If we multiply this by SST, it \"just works,\" and the arrays are broadcasted\n", - "properly:\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "(ds.sst * weights).dims" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "
\n", - " Warning: If the arrays being broadcasted share a dimension name, but have different coordinates,\n", - " they will first be aligned using Xarray's default align settings (including filling missing values with NaNs).\n", - " If that's not what you want, it's best to call align explicitly before broadcasting.\n", - "
\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Weighted Reductions\n", - "\n", - "We could imagine computing the weighted spatial mean of SST manually, like this:\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "sst_mean = (ds.sst * weights).sum(dim=(\"lon\", \"lat\")) / weights.sum(dim=\"lat\")\n", - "sst_mean.plot()\n", - "plt.title(\"This is wrong!\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "That would be wrong, however, because the denominator (`weights.sum(dim='lat')`)\n", - "needs to be expanded to include the `lon` dimension and modified to account for\n", - "the missing values (land points).\n", - "\n", - "In general, weighted reductions on multidimensional arrays are complicated. To\n", - "make it a bit easier, Xarray provides a mechanism for weighted reductions. It\n", - "does this by creating a special intermediate `DataArrayWeighted` object, to\n", - "which different reduction operations can applied.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "sst_weighted = ds.sst.weighted(weights)\n", - "sst_weighted" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "sst_weighted.mean(dim=(\"lon\", \"lat\")).plot()\n", - "plt.title(\"Correct Global Mean SST\")" - ] - }, { "cell_type": "markdown", "metadata": {}, From 50d119b096794834a3a3fb53a04593e1cf8c4bee Mon Sep 17 00:00:00 2001 From: Jessica Scheick Date: Fri, 10 Jun 2022 13:19:43 -0400 Subject: [PATCH 04/17] fix jupyter book build [path] errors --- _toc.yml | 4 ++-- workshops/online-tutorial-series/01_xarray_fundamentals.ipynb | 2 +- workshops/online-tutorial-series/03_computation.ipynb | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/_toc.yml b/_toc.yml index 9812c705..ab5b16a0 100644 --- a/_toc.yml +++ b/_toc.yml @@ -17,9 +17,9 @@ parts: - caption: Workshops chapters: - - file: workshops/oceanhackweek-2020/README + - file: workshops/oceanhackweek2020/README sections: - - file: workshops/oceanhackweek-2020/xarray-oceanhackweek20 + - file: workshops/oceanhackweek2020/xarray-oceanhackweek20 - file: workshops/thinking-like-xarray/README sections: - file: workshops/thinking-like-xarray/01-high-level-computation-patterns diff --git a/workshops/online-tutorial-series/01_xarray_fundamentals.ipynb b/workshops/online-tutorial-series/01_xarray_fundamentals.ipynb index da0f8014..6d7198c5 100644 --- a/workshops/online-tutorial-series/01_xarray_fundamentals.ipynb +++ b/workshops/online-tutorial-series/01_xarray_fundamentals.ipynb @@ -57,7 +57,7 @@ " - `Dataset`, which holds multiple variables that potentially share the same\n", " coordinates\n", "\n", - "![](../images/xarray-data-structures.png)\n" + "![](../../images/xarray-data-structures.png)\n" ] }, { diff --git a/workshops/online-tutorial-series/03_computation.ipynb b/workshops/online-tutorial-series/03_computation.ipynb index fc350df7..84ce61a0 100644 --- a/workshops/online-tutorial-series/03_computation.ipynb +++ b/workshops/online-tutorial-series/03_computation.ipynb @@ -101,7 +101,7 @@ "The image below t illustrates how performing an operation on arrays with\n", "differenty coordinates will result in automatic broadcasting\n", "\n", - "![](../images/broadcasting.png)\n", + "![](../../images/broadcasting.png)\n", "\n", "Credit: Stephan Hoyer --\n", "[xarray ECMWF Python workshop](https://docs.google.com/presentation/d/16CMY3g_OYr6fQplUZIDqVtG-SKZqsG8Ckwoj2oOqepU/edit#slide=id.g2b68f9254d_1_154)\n" From d7d166a0e08db92d3437e43c431825534743bfbe Mon Sep 17 00:00:00 2001 From: Jessica Scheick Date: Fri, 10 Jun 2022 13:25:40 -0400 Subject: [PATCH 05/17] fix more jupyterbook path errors --- scipy-tutorial/00_overview.ipynb | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/scipy-tutorial/00_overview.ipynb b/scipy-tutorial/00_overview.ipynb index 09f623d8..877ead2a 100644 --- a/scipy-tutorial/00_overview.ipynb +++ b/scipy-tutorial/00_overview.ipynb @@ -87,10 +87,13 @@ "\n", "The layout of the tutorial will be as follows:\n", "\n", - "1. [Introduction + Data structures for multi-dimensional data](./01_datastructures_and_io.ipynb)\n", - "1. [Working with labeled data](02_working_with_labeled_data.ipynb)\n", - "1. [Computation with Xarray](03_computation_with_xarray.ipynb)\n", - "1. [Plotting and Visualization](04_plotting_and_visualization.ipynb)\n", + "1. [Introduction + data structures for multi-dimensional data](./01_datastructures_and_io.ipynb)\n", + "1. [Working with labeled data](02.1_working_with_labeled_data.ipynb)\n", + "1. [Manipulating dimensions](02.2_manipulating_dimensions.ipynb)\n", + "1. [Aligning data objects](02.3_aligning_data_objects.ipynb)\n", + "1. [Computation with Xarray](03.1_computation_with_xarray.ipynb)\n", + "1. [Groupby with Xarray](03.2_groupby_with_xarray.ipynb)\n", + "1. [Plotting and visualization](04_plotting_and_visualization.ipynb)\n", "1. [Introduction to Dask](05_intro_to_dask.ipynb)\n", "1. [Dask and Xarray](06_xarray_and_dask.ipynb)\n", "\n", From 7c016659d9f41ac34935cb9bad9fcf7c2a7e0c8d Mon Sep 17 00:00:00 2001 From: Jessica Scheick Date: Fri, 10 Jun 2022 13:31:10 -0400 Subject: [PATCH 06/17] add titles to TOC --- _toc.yml | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/_toc.yml b/_toc.yml index ab5b16a0..bbc5aa04 100644 --- a/_toc.yml +++ b/_toc.yml @@ -7,10 +7,12 @@ parts: - file: scipy-tutorial/00_overview - file: scipy-tutorial/01_datastructures_and_io - file: scipy-tutorial/02.1_working_with_labeled_data - - file: scipy-tutorial/02.2_manipulating_dimensions - - file: scipy-tutorial/02.3_aligning_data_objects + sections: + - file: scipy-tutorial/02.2_manipulating_dimensions + - file: scipy-tutorial/02.3_aligning_data_objects - file: scipy-tutorial/03.1_computation_with_xarray - - file: scipy-tutorial/03.2_groupby_with_xarray + sections: + - file: scipy-tutorial/03.2_groupby_with_xarray - file: scipy-tutorial/04_plotting_and_visualization - file: scipy-tutorial/05_intro_to_dask - file: scipy-tutorial/06_xarray_and_dask @@ -31,16 +33,25 @@ parts: - file: workshops/scipy2022/README sections: - url: https://tutorial.xarray.dev/scipy-tutorial/00_overview + title: Introduction - url: https://tutorial.xarray.dev/scipy-tutorial/01_datastructures_and_io + title: Data Structures - file: workshops/scipy2022/cleaning_real_data - url: https://tutorial.xarray.dev/scipy-tutorial/02.1_working_with_labeled_data + title: Labeled Data - url: https://tutorial.xarray.dev/scipy-tutorial/02.2_manipulating_dimensions + title: Manipulating Dimensions - url: https://tutorial.xarray.dev/scipy-tutorial/02.3_aligning_data_objects + title: Data Alignment - url: https://tutorial.xarray.dev/scipy-tutorial/03.1_computation_with_xarray + title: Computation - url: https://tutorial.xarray.dev/scipy-tutorial/03.2_groupby_with_xarray + title: Groupby - url: https://tutorial.xarray.dev/scipy-tutorial/04_plotting_and_visualization.html + title: Plotting and Visualization - file: workshops/scipy2022/xarray_ecosystem - url: https://tutorial.xarray.dev/scipy-tutorial/06_xarray_and_dask + title: Xarray and Dask - caption: Reference chapters: From 7e016b64ef4646f3e2a5f86d6f17ccc346d1ef6e Mon Sep 17 00:00:00 2001 From: Jessica Scheick Date: Fri, 10 Jun 2022 13:43:47 -0400 Subject: [PATCH 07/17] fix more jupyterbook path errors --- workshops/online-tutorial-series/03_computation.ipynb | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/workshops/online-tutorial-series/03_computation.ipynb b/workshops/online-tutorial-series/03_computation.ipynb index 84ce61a0..a666c208 100644 --- a/workshops/online-tutorial-series/03_computation.ipynb +++ b/workshops/online-tutorial-series/03_computation.ipynb @@ -43,7 +43,7 @@ "metadata": {}, "outputs": [], "source": [ - "ds = xr.open_dataset(\"../data/sst.mnmean.nc\")\n", + "ds = xr.open_dataset(\"../../data/sst.mnmean.nc\")\n", "da = ds[\"sst\"]\n", "da" ] @@ -270,10 +270,15 @@ "metadata": {}, "source": [ "
\n", - "

Computation with xarray (extended version): Computation with xarray notebook

\n", - "

Plotting and visualization (extended version): Plotting and Visualization notebook

\n", + "

Computation with xarray (extended version): Computation with xarray notebook

\n", + "

Plotting and visualization (extended version): Plotting and Visualization notebook

\n", "
\n" ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [] } ], "metadata": { From 1e5da4ffd2c0727039da68c16f0feaa61d3fe419 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 10 Jun 2022 17:44:21 +0000 Subject: [PATCH 08/17] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- _toc.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/_toc.yml b/_toc.yml index bbc5aa04..06b4bc31 100644 --- a/_toc.yml +++ b/_toc.yml @@ -8,11 +8,11 @@ parts: - file: scipy-tutorial/01_datastructures_and_io - file: scipy-tutorial/02.1_working_with_labeled_data sections: - - file: scipy-tutorial/02.2_manipulating_dimensions - - file: scipy-tutorial/02.3_aligning_data_objects + - file: scipy-tutorial/02.2_manipulating_dimensions + - file: scipy-tutorial/02.3_aligning_data_objects - file: scipy-tutorial/03.1_computation_with_xarray sections: - - file: scipy-tutorial/03.2_groupby_with_xarray + - file: scipy-tutorial/03.2_groupby_with_xarray - file: scipy-tutorial/04_plotting_and_visualization - file: scipy-tutorial/05_intro_to_dask - file: scipy-tutorial/06_xarray_and_dask From bce6455da8d0500416ab5ff36331d3db8b7e669d Mon Sep 17 00:00:00 2001 From: dcherian Date: Sun, 12 Jun 2022 22:22:42 -0400 Subject: [PATCH 09/17] Fix thinking like Xarray title. --- workshops/thinking-like-xarray/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workshops/thinking-like-xarray/README.md b/workshops/thinking-like-xarray/README.md index 96f269e0..d97d720f 100644 --- a/workshops/thinking-like-xarray/README.md +++ b/workshops/thinking-like-xarray/README.md @@ -1,4 +1,4 @@ -# High-level computational patterns +# Thinking like Xarray Presented March 2022 for the [NCAR Python Seminar Series](https://ncar.github.io/esds/posts/2022/Thinking-with-Xarray/) by Deepak Cheerian From 021c75e43041b66fc27e53e3cdc32f2f8cf42d26 Mon Sep 17 00:00:00 2001 From: dcherian Date: Sun, 12 Jun 2022 22:22:28 -0400 Subject: [PATCH 10/17] Add dask material under "Advanced" Make dask section "Advanced" instead --- _toc.yml | 6 ++++-- .../xarray_and_dask.ipynb | 0 2 files changed, 4 insertions(+), 2 deletions(-) rename scipy-tutorial/06_xarray_and_dask.ipynb => advanced/xarray_and_dask.ipynb (100%) diff --git a/_toc.yml b/_toc.yml index 06b4bc31..e7a0debd 100644 --- a/_toc.yml +++ b/_toc.yml @@ -14,8 +14,10 @@ parts: sections: - file: scipy-tutorial/03.2_groupby_with_xarray - file: scipy-tutorial/04_plotting_and_visualization - - file: scipy-tutorial/05_intro_to_dask - - file: scipy-tutorial/06_xarray_and_dask + + - caption: Advanced + chapters: + - file: advanced/xarray_and_dask - caption: Workshops chapters: diff --git a/scipy-tutorial/06_xarray_and_dask.ipynb b/advanced/xarray_and_dask.ipynb similarity index 100% rename from scipy-tutorial/06_xarray_and_dask.ipynb rename to advanced/xarray_and_dask.ipynb From 63714e6fa2ddcdee8edf25acc7adf9414a8ae982 Mon Sep 17 00:00:00 2001 From: dcherian Date: Sun, 12 Jun 2022 22:31:44 -0400 Subject: [PATCH 11/17] Add computation patterns as "intermediate" material --- _toc.yml | 7 ++++++- .../01-high-level-computation-patterns.ipynb | 0 2 files changed, 6 insertions(+), 1 deletion(-) rename {workshops/thinking-like-xarray => intermediate}/01-high-level-computation-patterns.ipynb (100%) diff --git a/_toc.yml b/_toc.yml index e7a0debd..a78d3911 100644 --- a/_toc.yml +++ b/_toc.yml @@ -15,6 +15,10 @@ parts: - file: scipy-tutorial/03.2_groupby_with_xarray - file: scipy-tutorial/04_plotting_and_visualization + - caption: Intermediate + chapters: + - file: intermediate/01-high-level-computation-patterns + - caption: Advanced chapters: - file: advanced/xarray_and_dask @@ -26,7 +30,8 @@ parts: - file: workshops/oceanhackweek2020/xarray-oceanhackweek20 - file: workshops/thinking-like-xarray/README sections: - - file: workshops/thinking-like-xarray/01-high-level-computation-patterns + - url: https://tutorial.xarray.dev/intermediate/01-high-level-computation-patterns + title: High-level computation patterns - file: workshops/online-tutorial-series/README sections: - file: workshops/online-tutorial-series/01_xarray_fundamentals diff --git a/workshops/thinking-like-xarray/01-high-level-computation-patterns.ipynb b/intermediate/01-high-level-computation-patterns.ipynb similarity index 100% rename from workshops/thinking-like-xarray/01-high-level-computation-patterns.ipynb rename to intermediate/01-high-level-computation-patterns.ipynb From 946602d1e20c8e8c91b5d73414e4b0db8a0cf3ce Mon Sep 17 00:00:00 2001 From: dcherian Date: Sun, 12 Jun 2022 22:37:40 -0400 Subject: [PATCH 12/17] Rename "scipy-tutorial" to "fundamentals" --- _toc.yml | 34 ++++++------ .../00_overview.ipynb | 0 .../01_datastructures_and_io.ipynb | 0 .../02.1_working_with_labeled_data.ipynb | 0 .../02.2_manipulating_dimensions.ipynb | 0 .../02.3_aligning_data_objects.ipynb | 0 .../03.1_computation_with_xarray.ipynb | 0 .../03.2_groupby_with_xarray.ipynb | 0 .../04_plotting_and_visualization.ipynb | 0 .../05_intro_to_dask.ipynb | 0 {scipy-tutorial => fundamentals}/README.md | 0 fundamentals/dask-report-large-chunk.html | 54 +++++++++++++++++++ 12 files changed, 71 insertions(+), 17 deletions(-) rename {scipy-tutorial => fundamentals}/00_overview.ipynb (100%) rename {scipy-tutorial => fundamentals}/01_datastructures_and_io.ipynb (100%) rename {scipy-tutorial => fundamentals}/02.1_working_with_labeled_data.ipynb (100%) rename {scipy-tutorial => fundamentals}/02.2_manipulating_dimensions.ipynb (100%) rename {scipy-tutorial => fundamentals}/02.3_aligning_data_objects.ipynb (100%) rename {scipy-tutorial => fundamentals}/03.1_computation_with_xarray.ipynb (100%) rename {scipy-tutorial => fundamentals}/03.2_groupby_with_xarray.ipynb (100%) rename {scipy-tutorial => fundamentals}/04_plotting_and_visualization.ipynb (100%) rename {scipy-tutorial => fundamentals}/05_intro_to_dask.ipynb (100%) rename {scipy-tutorial => fundamentals}/README.md (100%) create mode 100644 fundamentals/dask-report-large-chunk.html diff --git a/_toc.yml b/_toc.yml index a78d3911..c18fbf2d 100644 --- a/_toc.yml +++ b/_toc.yml @@ -4,16 +4,16 @@ format: jb-book parts: - caption: Fundamentals chapters: - - file: scipy-tutorial/00_overview - - file: scipy-tutorial/01_datastructures_and_io - - file: scipy-tutorial/02.1_working_with_labeled_data + - file: fundamentals/00_overview + - file: fundamentals/01_datastructures_and_io + - file: fundamentals/02.1_working_with_labeled_data sections: - - file: scipy-tutorial/02.2_manipulating_dimensions - - file: scipy-tutorial/02.3_aligning_data_objects - - file: scipy-tutorial/03.1_computation_with_xarray + - file: fundamentals/02.2_manipulating_dimensions + - file: fundamentals/02.3_aligning_data_objects + - file: fundamentals/03.1_computation_with_xarray sections: - - file: scipy-tutorial/03.2_groupby_with_xarray - - file: scipy-tutorial/04_plotting_and_visualization + - file: fundamentals/03.2_groupby_with_xarray + - file: fundamentals/04_plotting_and_visualization - caption: Intermediate chapters: @@ -39,25 +39,25 @@ parts: - file: workshops/online-tutorial-series/03_computation - file: workshops/scipy2022/README sections: - - url: https://tutorial.xarray.dev/scipy-tutorial/00_overview + - url: https://tutorial.xarray.dev/fundamentals/00_overview title: Introduction - - url: https://tutorial.xarray.dev/scipy-tutorial/01_datastructures_and_io + - url: https://tutorial.xarray.dev/fundamentals/01_datastructures_and_io title: Data Structures - file: workshops/scipy2022/cleaning_real_data - - url: https://tutorial.xarray.dev/scipy-tutorial/02.1_working_with_labeled_data + - url: https://tutorial.xarray.dev/fundamentals/02.1_working_with_labeled_data title: Labeled Data - - url: https://tutorial.xarray.dev/scipy-tutorial/02.2_manipulating_dimensions + - url: https://tutorial.xarray.dev/fundamentals/02.2_manipulating_dimensions title: Manipulating Dimensions - - url: https://tutorial.xarray.dev/scipy-tutorial/02.3_aligning_data_objects + - url: https://tutorial.xarray.dev/fundamentals/02.3_aligning_data_objects title: Data Alignment - - url: https://tutorial.xarray.dev/scipy-tutorial/03.1_computation_with_xarray + - url: https://tutorial.xarray.dev/fundamentals/03.1_computation_with_xarray title: Computation - - url: https://tutorial.xarray.dev/scipy-tutorial/03.2_groupby_with_xarray + - url: https://tutorial.xarray.dev/fundamentals/03.2_groupby_with_xarray title: Groupby - - url: https://tutorial.xarray.dev/scipy-tutorial/04_plotting_and_visualization.html + - url: https://tutorial.xarray.dev/fundamentals/04_plotting_and_visualization.html title: Plotting and Visualization - file: workshops/scipy2022/xarray_ecosystem - - url: https://tutorial.xarray.dev/scipy-tutorial/06_xarray_and_dask + - url: https://tutorial.xarray.dev/fundamentals/06_xarray_and_dask title: Xarray and Dask - caption: Reference diff --git a/scipy-tutorial/00_overview.ipynb b/fundamentals/00_overview.ipynb similarity index 100% rename from scipy-tutorial/00_overview.ipynb rename to fundamentals/00_overview.ipynb diff --git a/scipy-tutorial/01_datastructures_and_io.ipynb b/fundamentals/01_datastructures_and_io.ipynb similarity index 100% rename from scipy-tutorial/01_datastructures_and_io.ipynb rename to fundamentals/01_datastructures_and_io.ipynb diff --git a/scipy-tutorial/02.1_working_with_labeled_data.ipynb b/fundamentals/02.1_working_with_labeled_data.ipynb similarity index 100% rename from scipy-tutorial/02.1_working_with_labeled_data.ipynb rename to fundamentals/02.1_working_with_labeled_data.ipynb diff --git a/scipy-tutorial/02.2_manipulating_dimensions.ipynb b/fundamentals/02.2_manipulating_dimensions.ipynb similarity index 100% rename from scipy-tutorial/02.2_manipulating_dimensions.ipynb rename to fundamentals/02.2_manipulating_dimensions.ipynb diff --git a/scipy-tutorial/02.3_aligning_data_objects.ipynb b/fundamentals/02.3_aligning_data_objects.ipynb similarity index 100% rename from scipy-tutorial/02.3_aligning_data_objects.ipynb rename to fundamentals/02.3_aligning_data_objects.ipynb diff --git a/scipy-tutorial/03.1_computation_with_xarray.ipynb b/fundamentals/03.1_computation_with_xarray.ipynb similarity index 100% rename from scipy-tutorial/03.1_computation_with_xarray.ipynb rename to fundamentals/03.1_computation_with_xarray.ipynb diff --git a/scipy-tutorial/03.2_groupby_with_xarray.ipynb b/fundamentals/03.2_groupby_with_xarray.ipynb similarity index 100% rename from scipy-tutorial/03.2_groupby_with_xarray.ipynb rename to fundamentals/03.2_groupby_with_xarray.ipynb diff --git a/scipy-tutorial/04_plotting_and_visualization.ipynb b/fundamentals/04_plotting_and_visualization.ipynb similarity index 100% rename from scipy-tutorial/04_plotting_and_visualization.ipynb rename to fundamentals/04_plotting_and_visualization.ipynb diff --git a/scipy-tutorial/05_intro_to_dask.ipynb b/fundamentals/05_intro_to_dask.ipynb similarity index 100% rename from scipy-tutorial/05_intro_to_dask.ipynb rename to fundamentals/05_intro_to_dask.ipynb diff --git a/scipy-tutorial/README.md b/fundamentals/README.md similarity index 100% rename from scipy-tutorial/README.md rename to fundamentals/README.md diff --git a/fundamentals/dask-report-large-chunk.html b/fundamentals/dask-report-large-chunk.html new file mode 100644 index 00000000..c4c76a7e --- /dev/null +++ b/fundamentals/dask-report-large-chunk.html @@ -0,0 +1,54 @@ + + + + + Dask Performance Report + + + + + + +
+ + + + + \ No newline at end of file From 66c2bdb63d8534f5a1894be0823e739807619026 Mon Sep 17 00:00:00 2001 From: dcherian Date: Sun, 12 Jun 2022 22:39:25 -0400 Subject: [PATCH 13/17] Fix typo --- workshops/scipy2022/xarray_ecosystem.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workshops/scipy2022/xarray_ecosystem.ipynb b/workshops/scipy2022/xarray_ecosystem.ipynb index ac57e933..722f2264 100644 --- a/workshops/scipy2022/xarray_ecosystem.ipynb +++ b/workshops/scipy2022/xarray_ecosystem.ipynb @@ -39,7 +39,7 @@ }, "outputs": [], "source": [ - "# rioxarray eample" + "# rioxarray example" ] }, { From 91fbe537c1689cbeb87d6947a2b2b5f0a5da891f Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 13 Jun 2022 02:43:34 +0000 Subject: [PATCH 14/17] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- fundamentals/dask-report-large-chunk.html | 22544 +++++++++++++++++++- 1 file changed, 22509 insertions(+), 35 deletions(-) diff --git a/fundamentals/dask-report-large-chunk.html b/fundamentals/dask-report-large-chunk.html index c4c76a7e..4f9694fd 100644 --- a/fundamentals/dask-report-large-chunk.html +++ b/fundamentals/dask-report-large-chunk.html @@ -1,54 +1,22528 @@ - + Dask Performance Report - - - + + + -
- +
+ - \ No newline at end of file + From 4d14c778d8b64358f37f5a2bee41051c30e5efe1 Mon Sep 17 00:00:00 2001 From: Deepak Cherian Date: Sun, 12 Jun 2022 20:44:12 -0600 Subject: [PATCH 15/17] Delete dask-report-large-chunk.html --- fundamentals/dask-report-large-chunk.html | 22528 -------------------- 1 file changed, 22528 deletions(-) delete mode 100644 fundamentals/dask-report-large-chunk.html diff --git a/fundamentals/dask-report-large-chunk.html b/fundamentals/dask-report-large-chunk.html deleted file mode 100644 index 4f9694fd..00000000 --- a/fundamentals/dask-report-large-chunk.html +++ /dev/null @@ -1,22528 +0,0 @@ - - - - - Dask Performance Report - - - - - - -
- - - - - From 326264fc462a2b3f16124251faf86d52475bbd73 Mon Sep 17 00:00:00 2001 From: dcherian Date: Sun, 12 Jun 2022 22:51:03 -0400 Subject: [PATCH 16/17] Minor improvements. --- intermediate/01-high-level-computation-patterns.ipynb | 10 +--------- workshops/thinking-like-xarray/README.md | 2 +- 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/intermediate/01-high-level-computation-patterns.ipynb b/intermediate/01-high-level-computation-patterns.ipynb index 9425a26c..6a969e87 100644 --- a/intermediate/01-high-level-computation-patterns.ipynb +++ b/intermediate/01-high-level-computation-patterns.ipynb @@ -10,15 +10,7 @@ "tags": [] }, "source": [ - "# Thinking like Xarray : High-level computational patterns\n", - "\n", - "**Deepak Cherian, CGD**\n", - "\n", - "March 9, 2022\n", - "\n", - "---\n", - "\n", - "This version includes additional material inserted during the live lecture.\n" + "# High-level computational patterns\n" ] }, { diff --git a/workshops/thinking-like-xarray/README.md b/workshops/thinking-like-xarray/README.md index d97d720f..7b26493e 100644 --- a/workshops/thinking-like-xarray/README.md +++ b/workshops/thinking-like-xarray/README.md @@ -2,7 +2,7 @@ Presented March 2022 for the [NCAR Python Seminar Series](https://ncar.github.io/esds/posts/2022/Thinking-with-Xarray/) by Deepak Cheerian -This 60-minute, intermediate-level lesson covers common high-level computing patterns +This 60-minute, intermediate-level lesson covers common high-level computing patterns. The version of the notebook here includes material inserted during the presentation. Video Recording: https://www.youtube.com/watch?v=TSw3GF_d2y8&list=PLNemzZpJM7lUu_iGP_lA2m7SeSUwKSIvR From 1fc446a9b7ef6325c4ed83de5263db10f350adb3 Mon Sep 17 00:00:00 2001 From: dcherian Date: Mon, 13 Jun 2022 23:18:34 -0400 Subject: [PATCH 17/17] Move Xarray in 45 minutes to overview section --- _toc.yml | 6 +++++- .../xarray-in-45-min.ipynb | 0 2 files changed, 5 insertions(+), 1 deletion(-) rename workshops/oceanhackweek2020/xarray-oceanhackweek20.ipynb => overview/xarray-in-45-min.ipynb (100%) diff --git a/_toc.yml b/_toc.yml index c18fbf2d..424f3aed 100644 --- a/_toc.yml +++ b/_toc.yml @@ -2,6 +2,9 @@ root: intro format: jb-book parts: + - caption: Overview + chapters: + - file: overview/xarray-in-45-min - caption: Fundamentals chapters: - file: fundamentals/00_overview @@ -27,7 +30,8 @@ parts: chapters: - file: workshops/oceanhackweek2020/README sections: - - file: workshops/oceanhackweek2020/xarray-oceanhackweek20 + - url: https://tutorial.xarray.dev/overview/xarray-in-45-min + title: Xarray in 45 minutes - file: workshops/thinking-like-xarray/README sections: - url: https://tutorial.xarray.dev/intermediate/01-high-level-computation-patterns diff --git a/workshops/oceanhackweek2020/xarray-oceanhackweek20.ipynb b/overview/xarray-in-45-min.ipynb similarity index 100% rename from workshops/oceanhackweek2020/xarray-oceanhackweek20.ipynb rename to overview/xarray-in-45-min.ipynb