diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..c98a745 --- /dev/null +++ b/.gitignore @@ -0,0 +1,6 @@ +.vscode +.ruff_cache +.pytest_cache +**/__pycache__/ +dist/ +metoppyenv/ \ No newline at end of file diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..2ed9e2e --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,34 @@ +# disable autofixing PRs, commenting "pre-commit.ci autofix" on a pull request triggers a autofix +ci: + autofix_prs: false +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v5.0.0 + hooks: + # standard end of line/end of file cleanup + - id: mixed-line-ending + - id: end-of-file-fixer + - id: trailing-whitespace + # ensure syntaxes are valid + - id: check-toml + - id: check-yaml + - repo: meta + # see https://pre-commit.com/#meta-hooks + hooks: + - id: check-hooks-apply + - id: check-useless-excludes + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.11.0 + hooks: + # lint & attempt to correct failures (e.g. pyupgrade) + - id: ruff + args: [--fix] + # compatible replacement for black + - id: ruff-format + - repo: https://github.com/python-jsonschema/check-jsonschema + rev: 0.32.1 + hooks: + # verify github syntaxes + - id: check-github-workflows + - id: check-dependabot +exclude: ^(broken-)?recipes/.*$ diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..71da954 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,11 @@ +# Changelog + +All notable changes to this project are documented in this file. + +# 0.1.0 (2025-??-??) + +First release of metoppy package. + +### Features + +- code for v0.1.0 diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 0000000..deae067 --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,128 @@ + +# Contributor Covenant Code of Conduct + +## Our Pledge + +We as members, contributors, and leaders pledge to make participation in our +community a harassment-free experience for everyone, regardless of age, body +size, visible or invisible disability, ethnicity, sex characteristics, gender +identity and expression, level of experience, education, socio-economic status, +nationality, personal appearance, race, caste, color, religion, or sexual +identity and orientation. + +We pledge to act and interact in ways that contribute to an open, welcoming, +diverse, inclusive, and healthy community. + +## Our Standards + +Examples of behavior that contributes to a positive environment for our +community include: + +- Demonstrating empathy and kindness toward other people +- Being respectful of differing opinions, viewpoints, and experiences +- Giving and gracefully accepting constructive feedback +- Accepting responsibility and apologizing to those affected by our mistakes, + and learning from the experience +- Focusing on what is best not just for us as individuals, but for the overall + community + +Examples of unacceptable behavior include: + +- The use of sexualized language or imagery, and sexual attention or advances of + any kind +- Trolling, insulting or derogatory comments, and personal or political attacks +- Public or private harassment +- Publishing others' private information, such as a physical or email address, + without their explicit permission +- Other conduct which could reasonably be considered inappropriate in a + professional setting + +## Enforcement Responsibilities + +Community leaders are responsible for clarifying and enforcing our standards of +acceptable behavior and will take appropriate and fair corrective action in +response to any behavior that they deem inappropriate, threatening, offensive, +or harmful. + +Community leaders have the right and responsibility to remove, edit, or reject +comments, commits, code, wiki edits, issues, and other contributions that are +not aligned to this Code of Conduct, and will communicate reasons for moderation +decisions when appropriate. + +## Scope + +This Code of Conduct applies within all community spaces, and also applies when +an individual is officially representing the community in public spaces. +Examples of representing our community include using an official email address, +posting via an official social media account, or acting as an appointed +representative at an online or offline event. + +## Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be +reported to the community leaders responsible for enforcement at +[support@europeanweather.cloud](mailto:support@europeanweather.cloud). +All complaints will be reviewed and investigated promptly and fairly. + +All community leaders are obligated to respect the privacy and security of the +reporter of any incident. + +## Enforcement Guidelines + +Community leaders will follow these Community Impact Guidelines in determining +the consequences for any action they deem in violation of this Code of Conduct: + +### 1. Correction + +**Community Impact**: Use of inappropriate language or other behavior deemed +unprofessional or unwelcome in the community. + +**Consequence**: A private, written warning from community leaders, providing +clarity around the nature of the violation and an explanation of why the +behavior was inappropriate. A public apology may be requested. + +### 2. Warning + +**Community Impact**: A violation through a single incident or series of +actions. + +**Consequence**: A warning with consequences for continued behavior. No +interaction with the people involved, including unsolicited interaction with +those enforcing the Code of Conduct, for a specified period of time. This +includes avoiding interactions in community spaces as well as external channels +like social media. Violating these terms may lead to a temporary or permanent +ban. + +### 3. Temporary Ban + +**Community Impact**: A serious violation of community standards, including +sustained inappropriate behavior. + +**Consequence**: A temporary ban from any sort of interaction or public +communication with the community for a specified period of time. No public or +private interaction with the people involved, including unsolicited interaction +with those enforcing the Code of Conduct, is allowed during this period. +Violating these terms may lead to a permanent ban. + +### 4. Permanent Ban + +**Community Impact**: Demonstrating a pattern of violation of community +standards, including sustained inappropriate behavior, harassment of an +individual, or aggression toward or disparagement of classes of individuals. + +**Consequence**: A permanent ban from any sort of public interaction within the +community. + +## Attribution + +This Code of Conduct is adapted from the +[Contributor Covenant](https://www.contributor-covenant.org/), version 2.1, +available at +. + +Community Impact Guidelines were inspired by +[Mozilla's code of conduct enforcement ladder](https://github.com/mozilla/inclusion). + +For answers to common questions about this code of conduct, see the FAQ at +. Translations are available at +. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..ec9c444 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,60 @@ +# How to Contribute + +This project is [licensed](./LICENSE) and accepts contributions via GitHub pull +requests. In this document, we outline some of the conventions on development +workflow, commit message formatting, contact points and other resources to make +it easier to get your contribution accepted. + +## Getting Started + +* Fork the repository on GitHub. +* Read the [README.md](./README.md) for usage/test instructions. +* Play with the project, submit bugs or patches. + +### Contribution Flow + +On the contributors' side: +1. Create a topic branch from the main brach to base your work on. +2. Make commits of logical units (checkout +[commit guidelines](#commit-guidelines) below). +3. Push changes to a topic branch in your fork of the repository. +4. Make sure to validate changes by running tests on a +EWC environment. +5. Submit a pull request to this repository, including details on +the steps necessary to reproduce your tests; assign maintainers for +review/approval. + +On the maintainers' side: + +1. Review, validate/test internally, and provide feedback prior to porting the +changes into a topic branch in the private upstream of this GitHub repository. +2. Upon approval, squash-merge changes in the upstream, making sure to leave +only one conventional commit as result from the merge; include full +acknowledgement of the contributors (i.e. list their GitHub handles) in the +body of the commit message. +3. Update the [CHANGELOG.md](./CHANGELOG.md), commit-tag it +to the main branch and mirror into GitHub. +4. Mark the relevant GitHub issue is as resolved. + +### Commit Guidelines + +This repository enforces +[conventional commits](https://www.conventionalcommits.org/en/v1.0.0/) +to mark breaking, major and minor code changes in accordance with the +[Semantic Versioning](https://semver.org/) standard: +- Commits that are merged to the main branch should always be prefixed by +specific keyword (i.e. `docs`, `style`, `feat`, `fix`, `refactor`, `ci`, +`chore` or `test`) +- Value is communicated to the end-users by three of the prefixes: + - `fix:` Patches a bug in your codebase. + - `feat:` Introduces a new feature to the codebase. + - `BREAKING CHANGE:` Introduces a breaking API change. A +`BREAKING CHANGE` can be part of commits of any type. + +## Reporting Security Vulnerabilities + +Due to their public nature, GitHub and RocketChat are not appropriate places +for reporting vulnerabilities. If you suspect you have found a security +vulnerability, please do not file a GitHub issue, but instead email +[support@europeanweather.cloud](mailto:support@europeanweather.cloud) with the +full details, including steps to reproduce the issue. diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..20b288e --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2025 EUMETSAT + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md index 0145a29..8087b48 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,164 @@ # MetopPy -Load native METOP products in Python. This is a wrapper around MetopDatasets.jl +Load native METOP products in Python. +It's a Python wrapper around [MetopDatasets.jl](https://github.com/eumetsat/MetopDatasets.jl) making it easy to install and use the package with Python. + +[MetopDatasets.jl](https://github.com/eumetsat/MetopDatasets.jl) is a package for reading products from the [METOP satellites](https://www.eumetsat.int/our-satellites/metop-series) using the native binary format specified for each product. The METOP satellites are part of the EUMETSAT-POLAR-SYSTEM (EPS) and have produced near real-time, global weather and climate observation since 2007. Learn more METOP and the data access on [EUMETSATs user-portal](https://user.eumetsat.int/dashboard). + +## Status +Metopdatasetpy is under development and is not ready for use yet. + +## Copyright and License +This code is licensed under MIT license. See file LICENSE for details on the usage and distribution terms. + +## Authors +* Simon Kok Lupemba - *Maintainer* - [EUMETSAT](http://www.eumetsat.int) +* Francesco Murdaca- *Contributor* - [EUMETSAT](http://www.eumetsat.int) + +## Installation +(This still have to be implemented) +```bash +pip install metoppy +``` + + +## Dependencies + +| Dependency | Version | License | Home URL | +|------|---------|---------|--------------| +| juliacall | >=0.9.14 | MIT License | https://juliapy.github.io/PythonCall.jl/stable/ | +| juliapkg | >=0.1.22 | MIT License | https://pypi.org/project/juliapkg/ | + + +## Build/Edit/Test Dependencies +The following dependencies are only required for building/editing/testing the software, they are not distributed with it: + +| Dependency | Version | License | Home URL | +|------|---------|---------|--------------| +| pytest | 8.4.1 | MIT License (MIT) | https://docs.pytest.org/en/latest | +| pytest-html | 4.1.1 | MIT License (MIT) | https://github.com/pytest-dev/pytest-html | +| pytest-mock | 3.14.1 | MIT License (MIT) | https://github.com/pytest-dev/pytest-mock | +| coverage | 7.10.5 | Apache Software License (Apache License Version 2.0) | https://github.com/nedbat/coveragepy | +| pre-commit | 4.3.0 | MIT License (MIT) | https://github.com/pre-commit/pre-commit | + +## Example + +1. Get test file + +```python +from pathlib import Path +from metoppy.metopreader import MetopReader + +metop_reader = MetopReader() +reduced_data_folder = metop_reader.get_test_data_artifact() +# ensure it's a Path object +reduced_data_folder = Path(reduced_data_folder) +reduced_data_files = [f for f in reduced_data_folder.iterdir() if f.is_file()] + +test_file_name = next((s for s in reduced_data_files if s.name.startswith("ASCA_SZO"))) +test_file_path = reduced_data_folder / test_file_name +ds = metop_reader.load_dataset(file_path=str(test_file_path)) +``` + +2. Check keys + +```python +keys = metop_reader.get_keys(ds) +print(list(keys)) +``` +
+ +Output of the print + +``` +['record_start_time', 'record_stop_time', 'degraded_inst_mdr', 'degraded_proc_mdr', 'utc_line_nodes', 'abs_line_number', 'sat_track_azi', 'as_des_pass', 'swath_indicator', 'latitude', 'longitude', 'sigma0_trip', 'kp', 'inc_angle_trip', 'azi_angle_trip', 'num_val_trip', 'f_kp', 'f_usable', 'f_land', 'lcr', 'flagfield'] +``` + +
+ +3. Display variable information + +```python +print(ds['latitude']) +``` +
+ +Output of the print + +``` +latitude (42 × 10) + Datatype: Union{Missing, Float64} (Int32) + Dimensions: xtrack × atrack + Attributes: + description = Latitude (-90 to 90 deg) + missing_value = Int32[-2147483648] + scale_factor = 1.0e-6 +``` + +
+ +4. Read variable + +```python +from juliacall import Main + +# Convert CFVariable to a full Julia Array +latitude_julia = Main.Array(ds['latitude']) # preserves the 2D shape + +# Convert to nested Python list +latitude_list = [ + [latitude_julia[i, j] for j in range(latitude_julia.size[1])] + for i in range(latitude_julia.size[0]) +] + +# Print first 5x5 elements +for row in latitude_list[:5]: + print(row[:5]) +``` +
+ +Output of the print + +``` +[71.101406, 70.99323199999999, 70.88307999999999, 70.770985, 70.65697999999999] +[71.298254, 71.18898999999999, 71.077743, 70.964546, 70.849434] +[71.494503, 71.384132, 71.271771, 71.157454, 71.041217] +[71.690134, 71.578638, 71.465144, 71.349689, 71.23231] +[71.885128, 71.772486, 71.65784099999999, 71.541231, 71.422691] +``` + +
+ + + +## Development + +Pre-requisite: Install podman or docker in your machine. + +1. Fork this repository and move into it +```bash +git clone XXX && cd XXX +``` + +2. Start container mounting the content +```bash +podman run -v ./:/usr/local/bin/metoppy -it python:3.12 /bin/bash +``` + +or + +```bash +docker run -v ./:/usr/local/bin/metoppy -it python:3.12 /bin/bash +``` + +3. Move to the repository and install the package for testing +``` +cd /usr/local/bin/metoppy && pip install -e . +``` + +4. Modify the local code and test in the container. + +``` +python3 metoppy/tests/test_get_test_data_artifact.py +``` + +5. When you are happy, push code to your fork and open a MR (Gitlab) or PR (Github) diff --git a/RELEASING.md b/RELEASING.md new file mode 100644 index 0000000..a73eda4 --- /dev/null +++ b/RELEASING.md @@ -0,0 +1,65 @@ +# Releasing metoppy + +1. checkout main branch +2. pull from repo +3. run the unittests +4. Update the `CHANGELOG.md` file. +5. Create a tag with the new version number, starting with a 'v', eg: + + ``` + git tag -a v -m "Version " + ``` + + For example if the previous tag was `v0.9.0` and the new release is a + patch release, do: + + ``` + git tag -a v0.9.1 -m "Version 0.9.1" + ``` + + See [semver.org](http://semver.org/) on how to write a version number. + + +6. push changes to github `git push --follow-tags` +7. Verify github action unittests passed. +8. Create a "Release" on GitHub by going to + https://github.com/eumetsat/MetopDatasets.jl/releases and clicking "Draft a new release". + On the next page enter the newly created tag in the "Tag version" field, + "Version X.Y.Z" in the "Release title" field, and paste the markdown from + the changelog (the portion under the version section header) in the + "Describe this release" box. Finally click "Publish release". + +9. Now you can start the process to release on PyPI (only admins) + +9.1 Build package + +Now generate the distribution. To build the package, use PyPA build. + +1. Install the build tool +```bash +pip install -q build +``` + +```bash +python3 -m build +``` + +you will end up with a `/dist` file. The .whl file and .tar.gz can then be distributed and installed or pushed to PyPI. + +9.2 Push package to PyPI + +Install twine +```bash +pip install twine +``` + +If you want to test first, use TestPyPI: +```bash +twine upload --repository testpypi dist/* +``` + +To upload your package to PyPI, use Twine: +```bash +twine upload dist/* +``` +You'll be prompted for your PyPI username & password. diff --git a/metoppy/__init__.py b/metoppy/__init__.py new file mode 100644 index 0000000..c239af3 --- /dev/null +++ b/metoppy/__init__.py @@ -0,0 +1,21 @@ +#!/usr/bin/env python +# +# Package Name: metoppy +# Author: Simon Kok Lupemba, Francesco Murdaca +# License: MIT License +# Copyright (c) 2025 EUMETSAT + +# This package is licensed under the MIT License. +# See the LICENSE file for more details. + +"""Metoppy package initialization.""" + +# Optional: version info +__version__ = "0.1.0" + +# Import main classes/functions +from .metopreader import MetopReader + +__all__ = [ + MetopReader.__name__, +] diff --git a/metoppy/juliapkg.json b/metoppy/juliapkg.json new file mode 100644 index 0000000..b9a5daa --- /dev/null +++ b/metoppy/juliapkg.json @@ -0,0 +1,9 @@ +{ + "julia": "^1.10", + "packages": { + "MetopDatasets": { + "uuid": "0c26954c-4046-4b98-a13c-f9377ca4b9b7", + "version": "0.2.0" + } + } +} diff --git a/metoppy/metopreader.py b/metoppy/metopreader.py new file mode 100644 index 0000000..2c84fad --- /dev/null +++ b/metoppy/metopreader.py @@ -0,0 +1,76 @@ +#!/usr/bin/env python +# +# Package Name: metoppy +# Author: Simon Kok Lupemba, Francesco Murdaca +# License: MIT License +# Copyright (c) 2025 EUMETSAT + +# This package is licensed under the MIT License. +# See the LICENSE file for more details. + + +"""MetopDatasets.jl Python wrapper class: MetopReader.""" + +from juliacall import Main + + +class MetopReader: + """Python wrapper class for MetopDatasets.jl Julia package.""" + + def __init__(self): + """ + Initialize the MetopReader by loading the MetopDatasets.jl package + into the Julia Main environment and caching references to its functions. + """ + # Import Julia package installed via juliapkg.json + Main.seval("import MetopDatasets") + # Store module and commonly used functions + self._keys = Main.MetopDatasets.keys + self._load_dataset = Main.MetopDatasets.MetopDataset + self._get_test_data_artifact = Main.MetopDatasets.get_test_data_artifact + + def get_keys(self, dataset): + """ + Return the available keys from a given MetopDataset. + + Parameters + ---------- + dataset : Julia object + A dataset object created by MetopDatasets.MetopDataset. + + Returns + ------- + list + The list of keys available in the dataset. + """ + return self._keys(dataset) + + def load_dataset(self, file_path: str): + """ + Load a dataset from a record path using MetopDatasets.MetopDataset. + + Parameters + ---------- + file_path : str + Path to the dataset record. + + Returns + ------- + Julia object + A MetopDataset object loaded from the provided path. + """ + try: + return self._load_dataset(file_path) + except Exception as e: + raise RuntimeError(f"Failed to load dataset: {file_path}") from e + + def get_test_data_artifact(self): + """ + Retrieve the test dataset artifact from MetopDatasets. + + Returns + ------- + Julia object + A MetopDataset object containing test data for validation or demo purposes. + """ + return self._get_test_data_artifact() diff --git a/metoppy/tests/__init__.py b/metoppy/tests/__init__.py new file mode 100644 index 0000000..17027e3 --- /dev/null +++ b/metoppy/tests/__init__.py @@ -0,0 +1,11 @@ +#!/usr/bin/env python +# +# Package Name: metoppy +# Author: Simon Kok Lupemba, Francesco Murdaca +# License: MIT License +# Copyright (c) 2025 EUMETSAT + +# This package is licensed under the MIT License. +# See the LICENSE file for more details. + +"""Tests for metoppy.""" diff --git a/metoppy/tests/test_get_test_data_artifact.py b/metoppy/tests/test_get_test_data_artifact.py new file mode 100644 index 0000000..733e78e --- /dev/null +++ b/metoppy/tests/test_get_test_data_artifact.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python +# +# Package Name: metoppy +# Author: Simon Kok Lupemba, Francesco Murdaca +# License: MIT License +# Copyright (c) 2025 EUMETSAT + +# This package is licensed under the MIT License. +# See the LICENSE file for more details. + +"""Test file.""" + +# TODO: Convert to pytest +from pathlib import Path +from juliacall import Main +from metoppy.metopreader import MetopReader + +metop_reader = MetopReader() +reduced_data_folder = metop_reader.get_test_data_artifact() +# ensure it's a Path object +reduced_data_folder = Path(reduced_data_folder) +reduced_data_files = [f for f in reduced_data_folder.iterdir() if f.is_file()] + +test_file_name = next((s for s in reduced_data_files if s.name.startswith("ASCA_SZO"))) +test_file_path = reduced_data_folder / test_file_name +ds = metop_reader.load_dataset(file_path=str(test_file_path)) + +keys = metop_reader.get_keys(ds) +print(list(keys)) + +print(ds["latitude"]) + +# Convert CFVariable to a full Julia Array +latitude_julia = Main.Array(ds["latitude"]) # preserves the 2D shape + +# Convert to nested Python list +latitude_list = [ + [latitude_julia[i, j] for j in range(latitude_julia.size[1])] + for i in range(latitude_julia.size[0]) +] + +# Print first 5x5 elements +for row in latitude_list[:5]: + print(row[:5]) diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..10d5527 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,54 @@ +[project] +name = "metoppy" +version = "0.1.0" +description = "Python package wrapper of MetopDatasets.jl Julia package for reading products from the METOP satellites." +authors = [ + { name = "Simon Kok Lupemba", email = "simon.koklupemba@eumetsat.int" }, + { name = "Francesco Murdaca", email = "francesco.murdaca@eumetsat.int" } +] +maintainers = [ + { name = "Simon Kok Lupemba", email = "simon.koklupemba@eumetsat.int" } +] +dependencies = [ + "juliacall>=0.9.14", + "juliapkg>=0.1.22", +] +readme = "README.md" +requires-python = ">=3.11" +license = { file = "LICENSE" } +classifiers = [ + "Development Status :: 2 - Pre-Alpha", + "Intended Audience :: Science/Research", + "License :: OSI Approved :: MIT License", + "Operating System :: OS Independent", + "Programming Language :: Python", + "Topic :: Scientific/Engineering" +] + +[project.urls] +Homepage = "https://github.com/eumetsat/MetopDatasets.jl" +"Bug Tracker" = "https://github.com/eumetsat/MetopDatasets.jl/issues" +Documentation = "https://github.com/eumetsat/MetopDatasets.jl/blob/main/README.md" +"Source Code" = "https://github.com/eumetsat/MetopDatasets.jl" +Organization = "https://github.com/eumetsat" +"Release Notes" = "https://github.com/eumetsat/MetopDatasets.jl/blob/main/CHANGELOG.md" + + +[project.optional-dependencies] +test = [ + "pytest==8.4.1", + "pytest-html==4.1.1", + "pytest-mock==3.14.1", + "coverage==7.10.5", + "pre-commit==4.3.0", +] + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[tool.hatch.build.targets.sdist] +include = ["metoppy", "README.md", "LICENSE"] + +[tool.hatch.build.targets.wheel] +packages = ["metoppy"]