From b20206043af05f8476a1aa1bedbf7ed38e332fc5 Mon Sep 17 00:00:00 2001 From: Stephen L Arnold Date: Tue, 16 Feb 2021 13:46:51 -0800 Subject: [PATCH 01/10] new: add updated conda recipe and corresponding ci workflow Signed-off-by: Stephen L Arnold --- .github/workflows/conda.yml | 55 +++++++++++++++++++++++++++++++++++++ conda.recipe/meta.yaml | 52 +++++++++++++++++++++++++++++++++++ 2 files changed, 107 insertions(+) create mode 100644 .github/workflows/conda.yml create mode 100644 conda.recipe/meta.yaml diff --git a/.github/workflows/conda.yml b/.github/workflows/conda.yml new file mode 100644 index 0000000..5ec44f3 --- /dev/null +++ b/.github/workflows/conda.yml @@ -0,0 +1,55 @@ +name: Conda + +on: + workflow_dispatch: + pull_request: + push: + branches: + - master + +jobs: + build: + strategy: + fail-fast: false + matrix: + platform: [ubuntu-latest, windows-2016, macos-latest] + python-version: [3.6, 3.7, 3.8, 3.9] + + runs-on: ${{ matrix.platform }} + + # The setup-miniconda action needs this to activate miniconda + defaults: + run: + shell: "bash -l {0}" + + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + + - name: Cache conda + uses: actions/cache@v1 + with: + path: ~/conda_pkgs_dir + key: ${{matrix.os}}-conda-pkgs-${{hashFiles('**/conda.recipe/meta.yaml')}} + + - name: Get conda + uses: conda-incubator/setup-miniconda@v2 + with: + python-version: ${{ matrix.python-version }} + channels: conda-forge + channel-priority: strict + use-only-tar-bz2: true + auto-activate-base: true + + - name: Prepare + run: conda install conda-build conda-verify pytest hypothesis + + - name: Build + run: conda build conda.recipe + + - name: Install + run: conda install -c ${CONDA_PREFIX}/conda-bld/ datrie + + - name: Test + run: pytest -v diff --git a/conda.recipe/meta.yaml b/conda.recipe/meta.yaml new file mode 100644 index 0000000..181539c --- /dev/null +++ b/conda.recipe/meta.yaml @@ -0,0 +1,52 @@ +{% set name = "datrie" %} +{% set version = "0.8.3.dev0" %} + +package: + name: {{ name|lower }} + version: {{ version }} + +source: + path: .. + +build: + number: 0 + script: {{ PYTHON }} -m pip install . -vv + +requirements: + build: + - {{ compiler('c') }} + - {{ compiler('cxx') }} + host: + - python + - cmake >=3.15 + - pybind11 + - ninja + - cython + - pip + + run: + - python + +test: + requires: + - pytest + - hypothesis + imports: + - datrie + source_files: + - tests + commands: + - python -m pytest + +about: + home: "https://github.com/pytries" + license: LGPL-2.0-or-later + license_family: LGPL + license_file: COPYING + summary: "Fast, efficiently stored Trie for Python. Uses libdatrie." + doc_url: "https://github.com/pytries/datrie/blob/master/README.rst" + dev_url: "https://github.com/pytries/datrie" + +extra: + recipe-maintainers: + - sarnold From 939fa833ba8bb755b6ac86dd9d11496a37cf3295 Mon Sep 17 00:00:00 2001 From: Stephen L Arnold Date: Tue, 16 Feb 2021 17:24:38 -0800 Subject: [PATCH 02/10] chg: update tox cfg and adjust ci (disable conda, update checkout) Signed-off-by: Stephen L Arnold --- .github/workflows/ci.yml | 8 +++++--- .github/workflows/conda.yml | 9 +++++---- .github/workflows/release.yml | 1 + .github/workflows/wheels.yml | 1 + tox.ini | 6 +++--- 5 files changed, 15 insertions(+), 10 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 76a4e1a..96c0836 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -32,6 +32,7 @@ jobs: - uses: actions/checkout@v2 with: fetch-depth: 0 + submodules: true - uses: actions/setup-python@v2 with: @@ -47,13 +48,13 @@ jobs: run: | brew install ninja - - name: Prepare compiler environment for ${{ matrix.os }} + - name: Prepare build environment for ${{ matrix.os }} if: runner.os == 'Windows' uses: ilammy/msvc-dev-cmd@v1 with: arch: x64 - - name: Set have package true for ${{ matrix.os }} + - name: Set GITHUB_ENV vars for ${{ matrix.os }} if: runner.os == 'Linux' run: | echo "HAVE_LIBDATRIE_PKG=TRUE" >> $GITHUB_ENV @@ -68,7 +69,8 @@ jobs: - name: Test in place # windows does not like build_ext -i or removing previous build - if: runner.os != 'Windows' + # (via Path inside tox env) + #if: runner.os != 'Windows' run: | tox -e py diff --git a/.github/workflows/conda.yml b/.github/workflows/conda.yml index 5ec44f3..2cfc7d1 100644 --- a/.github/workflows/conda.yml +++ b/.github/workflows/conda.yml @@ -2,10 +2,10 @@ name: Conda on: workflow_dispatch: - pull_request: - push: - branches: - - master + #pull_request: + #push: + # branches: + # - master jobs: build: @@ -26,6 +26,7 @@ jobs: - uses: actions/checkout@v2 with: fetch-depth: 0 + submodules: true - name: Cache conda uses: actions/cache@v1 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 5892b9f..3921b46 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -18,6 +18,7 @@ jobs: - uses: actions/checkout@v2 with: fetch-depth: 0 + submodules: true - uses: actions/setup-python@v2 name: Install Python diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index fe7d296..ee9bc97 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -19,6 +19,7 @@ jobs: - uses: actions/checkout@v2 with: fetch-depth: 0 + submodules: true - uses: actions/setup-python@v2 name: Install Python diff --git a/tox.ini b/tox.ini index c0e0424..71c6424 100644 --- a/tox.ini +++ b/tox.ini @@ -23,19 +23,18 @@ passenv = PIP_DOWNLOAD_CACHE setenv = - PYTHONPATH=. + PYTHONPATH = {toxinidir} + #PYTHONPATH = {env:PYTHONPATH}{:}{toxinidir} deps = pip>=20.0.1 cython>=0.20 - path pytest hypothesis commands = python setup.py build_ext --inplace python -m pytest [] - python -c "import path; path.Path('build').rmtree_p()" [testenv:dev] skip_install = true @@ -56,6 +55,7 @@ deps = path commands= + # this command breaks on windows (permission error) python -c "import path; path.Path('build').rmtree_p()" pip install -e .[test] flake8 src/ From 85c8c05c1cc37c2cde1465985c7528bbf003a6f3 Mon Sep 17 00:00:00 2001 From: Stephen L Arnold Date: Tue, 16 Feb 2021 17:44:07 -0800 Subject: [PATCH 03/10] chg: restore git submodule, set commit d1dfdb8 (uses github relative path) Signed-off-by: Stephen L Arnold --- .gitmodules | 3 +++ libdatrie | 1 + 2 files changed, 4 insertions(+) create mode 100644 .gitmodules create mode 160000 libdatrie diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..0a0768f --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "libdatrie"] + path = libdatrie + url = ../../tlwg/libdatrie.git diff --git a/libdatrie b/libdatrie new file mode 160000 index 0000000..d1dfdb8 --- /dev/null +++ b/libdatrie @@ -0,0 +1 @@ +Subproject commit d1dfdb831093892541cae46eba82c46aec94f726 From 2a4186d0d6e8cfb3d1e92f7bc276306c5ae7cffe Mon Sep 17 00:00:00 2001 From: Stephen L Arnold Date: Tue, 16 Feb 2021 17:58:15 -0800 Subject: [PATCH 04/10] chg: pkg: switch cmake bits to look for submodule hdrs/srcs Signed-off-by: Stephen L Arnold --- CMakeLists.txt | 12 +----------- src/CMakeLists.txt | 6 +++--- 2 files changed, 4 insertions(+), 14 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 18e2530..63d24a4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -30,17 +30,7 @@ if(USE_LIBDATRIE_PKG AND NOT Datrie_FOUND) endif() if(NOT USE_LIBDATRIE_PKG) - message(STATUS "Fetching libdatrie from github") - # Fetch libdatrie - include(FetchContent) - - FetchContent_Declare( - libdatrie - GIT_REPOSITORY https://github.com/tlwg/libdatrie - GIT_TAG v0.2.13 - ) - FetchContent_MakeAvailable(libdatrie) - # this gets us the package source directory + message(STATUS "Using libdatrie submodule") endif() find_package(pybind11 CONFIG) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index eeed7eb..fe3864c 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -21,11 +21,11 @@ add_custom_command(OUTPUT ${cython_output} COMMENT "Cythonizing extension ${cython_src}") if(NOT USE_LIBDATRIE_PKG) - # use the locally cloned source from FetchContent - set(DATRIE_INCLUDE_DIR "${libdatrie_SOURCE_DIR}") + # use the local git submodule + set(DATRIE_INCLUDE_DIR "${PROJECT_SOURCE_DIR}/libdatrie") file(GLOB_RECURSE DATRIE_SOURCES LIST_DIRECTORIES true - "${libdatrie_SOURCE_DIR}/datrie/*.c") + "${DATRIE_INCLUDE_DIR}/datrie/*.c") list(APPEND cython_output ${DATRIE_SOURCES}) include_directories(${DATRIE_INCLUDE_DIR}) endif() From 9df70c26cb37c7e9edec00b15cd1ebe8f712b660 Mon Sep 17 00:00:00 2001 From: Stephen L Arnold Date: Tue, 16 Feb 2021 18:15:53 -0800 Subject: [PATCH 05/10] cleanup commit !minor Signed-off-by: Stephen L Arnold --- .github/workflows/ci.yml | 3 --- CMakeLists.txt | 2 +- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 96c0836..e36b37c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -68,9 +68,6 @@ jobs: packages: libdatrie-dev pybind11-dev ninja-build - name: Test in place - # windows does not like build_ext -i or removing previous build - # (via Path inside tox env) - #if: runner.os != 'Windows' run: | tox -e py diff --git a/CMakeLists.txt b/CMakeLists.txt index 63d24a4..eca1c03 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -30,7 +30,7 @@ if(USE_LIBDATRIE_PKG AND NOT Datrie_FOUND) endif() if(NOT USE_LIBDATRIE_PKG) - message(STATUS "Using libdatrie submodule") + message(STATUS "Trying libdatrie submodule") endif() find_package(pybind11 CONFIG) From 8d3fb4956c5a0a78946c95769bc93d4f9966c51b Mon Sep 17 00:00:00 2001 From: Stephen L Arnold Date: Tue, 16 Feb 2021 18:56:21 -0800 Subject: [PATCH 06/10] chg: pkg: let cmake init the submodule if empty Signed-off-by: Stephen L Arnold --- CMakeLists.txt | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index eca1c03..1050543 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,6 +1,7 @@ cmake_minimum_required(VERSION 3.15...3.18) option(PY_DEBUG "Set if python being linked is a Py_DEBUG build" OFF) +option(GIT_SUBMODULE "Check submodules during build" ON) option(USE_LIBDATRIE_PKG "Use OS-provided libdatrie package") if(DEFINED ENV{HAVE_LIBDATRIE_PKG}) set(USE_LIBDATRIE_PKG "$ENV{HAVE_LIBDATRIE_PKG}") @@ -31,6 +32,25 @@ endif() if(NOT USE_LIBDATRIE_PKG) message(STATUS "Trying libdatrie submodule") + find_package(Git QUIET) + if(GIT_FOUND AND EXISTS "${PROJECT_SOURCE_DIR}/.git") + # Update submodules as needed + if(GIT_SUBMODULE) + if(NOT EXISTS "${PROJECT_SOURCE_DIR}/libdatrie/datrie-0.2.pc.in") + message(STATUS "Submodule update") + execute_process(COMMAND ${GIT_EXECUTABLE} submodule update --init --recursive + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} + RESULT_VARIABLE GIT_SUBMOD_RESULT) + if(NOT GIT_SUBMOD_RESULT EQUAL "0") + message(FATAL_ERROR "git submodule update --init failed with ${GIT_SUBMOD_RESULT}, please checkout submodules") + endif() + endif() + endif() + endif() + + if(NOT EXISTS "${PROJECT_SOURCE_DIR}/libdatrie/datrie-0.2.pc.in") + message(FATAL_ERROR "The submodules were not downloaded! GIT_SUBMODULE was turned off or failed. Please update submodules and try again.") + endif() endif() find_package(pybind11 CONFIG) From a2090bb9b956460152189e11e6459f114c38b4a4 Mon Sep 17 00:00:00 2001 From: Stephen L Arnold Date: Tue, 16 Feb 2021 19:01:52 -0800 Subject: [PATCH 07/10] remove bad submodle url Signed-off-by: Stephen L Arnold --- .gitmodules | 3 --- libdatrie | 1 - 2 files changed, 4 deletions(-) delete mode 100644 .gitmodules delete mode 160000 libdatrie diff --git a/.gitmodules b/.gitmodules deleted file mode 100644 index 0a0768f..0000000 --- a/.gitmodules +++ /dev/null @@ -1,3 +0,0 @@ -[submodule "libdatrie"] - path = libdatrie - url = ../../tlwg/libdatrie.git diff --git a/libdatrie b/libdatrie deleted file mode 160000 index d1dfdb8..0000000 --- a/libdatrie +++ /dev/null @@ -1 +0,0 @@ -Subproject commit d1dfdb831093892541cae46eba82c46aec94f726 From efe29b6bde1c7d31c4b7c83f3e1187273a8f9d16 Mon Sep 17 00:00:00 2001 From: Stephen L Arnold Date: Tue, 16 Feb 2021 19:09:07 -0800 Subject: [PATCH 08/10] re-add submodule so it uses https instead of git@ Signed-off-by: Stephen L Arnold --- .gitmodules | 3 +++ libdatrie | 1 + 2 files changed, 4 insertions(+) create mode 100644 .gitmodules create mode 160000 libdatrie diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..0a0768f --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "libdatrie"] + path = libdatrie + url = ../../tlwg/libdatrie.git diff --git a/libdatrie b/libdatrie new file mode 160000 index 0000000..fc571ea --- /dev/null +++ b/libdatrie @@ -0,0 +1 @@ +Subproject commit fc571ea08f84c59c9950b3e595ed862aa9aabea6 From 2c567e85f01a5dec400aac0a4f362b3684286c33 Mon Sep 17 00:00:00 2001 From: Stephen L Arnold Date: Tue, 16 Feb 2021 19:16:21 -0800 Subject: [PATCH 09/10] fix: relative path uses parent url style, make it https instead Signed-off-by: Stephen L Arnold --- .gitmodules | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitmodules b/.gitmodules index 0a0768f..3986f13 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,3 @@ [submodule "libdatrie"] path = libdatrie - url = ../../tlwg/libdatrie.git + url = https://github.com/tlwg/libdatrie.git From c0b2ee3b87af37c70277a3fab97d6418dc077a88 Mon Sep 17 00:00:00 2001 From: Stephen L Arnold Date: Tue, 16 Feb 2021 19:39:44 -0800 Subject: [PATCH 10/10] fix: reset submodule at upstream commit d1dfdb8 Signed-off-by: Stephen L Arnold --- libdatrie | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libdatrie b/libdatrie index fc571ea..d1dfdb8 160000 --- a/libdatrie +++ b/libdatrie @@ -1 +1 @@ -Subproject commit fc571ea08f84c59c9950b3e595ed862aa9aabea6 +Subproject commit d1dfdb831093892541cae46eba82c46aec94f726