diff --git a/.github/workflows/pypi.yaml b/.github/workflows/pypi.yaml new file mode 100644 index 0000000..15dff36 --- /dev/null +++ b/.github/workflows/pypi.yaml @@ -0,0 +1,20 @@ +name: PyPI + +on: + release: + types: + - created + +jobs: + job: + name: PyPI + runs-on: ubuntu-latest + env: + POETRY_PYPI_TOKEN_PYPI: ${{ secrets.PYPI_TOKEN }} + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: "3.12" + - name: Publish package to PyPI + run: pip install poetry && poetry publish --build diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml new file mode 100644 index 0000000..e52490d --- /dev/null +++ b/.github/workflows/tests.yaml @@ -0,0 +1,33 @@ +name: Tests + +on: + push: + branches: + - main + pull_request: + branches: + - main + +jobs: + job: + name: Test (Python ${{ matrix.python }}) + runs-on: ubuntu-latest + env: + POETRY_VIRTUALENVS_CREATE: false + strategy: + fail-fast: false + matrix: + python: ["3.9", "3.10", "3.11", "3.12"] + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python }} + - name: Install project dependencies + run: pip install poetry && poetry install + - name: Test code's formatting (Black) + run: black --check deshima_rawdata tests + - name: Test code's typing (Pyright) + run: pyright deshima_rawdata tests + - name: Test code's execution (pytest) + run: pytest -v tests diff --git a/README.md b/README.md index 0ab81d8..c84c143 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,17 @@ # rawdata -:truck: DESHIMA raw data and downloader package +DESHIMA raw data and downloader package + +# Usage + +``` +$ pip install deshima-rawdata +$ deshima-rawdata download +``` + +# List of raw data + +| Observation ID | File name | Source | Observation type | +| --- | --- | --- | --- | +| 20231108052231 | 20231108052231.tar.gz | Jupiter | contmap | +| 20231109015146 | 20231109015146.tar.gz | Jupiter | zscan | +| 20231109060113 | 20231109060113.tar.gz | Blank sky | skydip | diff --git a/data/20231108052231.tar.gz b/data/20231108052231.tar.gz new file mode 100644 index 0000000..01746af Binary files /dev/null and b/data/20231108052231.tar.gz differ diff --git a/data/20231109015146.tar.gz b/data/20231109015146.tar.gz new file mode 100644 index 0000000..27e8ea5 Binary files /dev/null and b/data/20231109015146.tar.gz differ diff --git a/data/20231109060113.tar.gz b/data/20231109060113.tar.gz new file mode 100644 index 0000000..e6da27d Binary files /dev/null and b/data/20231109060113.tar.gz differ diff --git a/deshima_rawdata/cli.py b/deshima_rawdata/cli.py index bef05dc..bb7cbcd 100644 --- a/deshima_rawdata/cli.py +++ b/deshima_rawdata/cli.py @@ -2,6 +2,7 @@ # standard library +import tarfile from pathlib import Path @@ -14,6 +15,7 @@ # constants CHUNK_SIZE = 1024 +DEFAULT_TAG = f"v{__version__}" GITHUB_URL = "https://raw.githubusercontent.com/deshima-dev/rawdata" @@ -22,27 +24,31 @@ def download( /, *, dir: Path = Path(), - progress: bool = True, + extract: bool = False, + progress: bool = False, + tag: str = DEFAULT_TAG, ) -> Path: """Download DESHIMA raw data for given observation ID. Args: obsid: Observation ID (YYYYmmddHHMMSS). dir: Directory where the raw data is saved. + extract: Whether to extract the raw data. progress: Whether to show a progress bar. + tag: Git tag (or branch) of the raw data. Returns: Path of the downloaded raw data. """ - url = f"{GITHUB_URL}/v{__version__}/data/{obsid}.tar.gz" + url = f"{GITHUB_URL}/{tag}/data/{obsid}.tar.gz" if not (response := get(url, stream=True)).ok: response.raise_for_status() bar_options = { "disable": not progress, - "total": int(response.headers.get("content-length", 0)), + "total": int(response.headers["content-length"]), "unit": "B", "unit_scale": True, } @@ -53,7 +59,15 @@ def download( bar.update(len(data)) f.write(data) - return data_path + if not extract: + return data_path + + with tarfile.open(data_path, "r:gz") as tar: + tar.extractall(data_path.parent) + dir_name = tar.getnames()[0] + + data_path.unlink(True) + return data_path.parent / dir_name def main() -> None: diff --git a/tests/test_cli.py b/tests/test_cli.py new file mode 100644 index 0000000..1d89018 --- /dev/null +++ b/tests/test_cli.py @@ -0,0 +1,2 @@ +def test_download() -> None: + pass