diff --git a/.circleci/config.yml b/.circleci/config.yml index f3874eb2c7..95a0473a37 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -715,9 +715,6 @@ jobs: steps: - checkout - install-rustup - - run: - name: Run uniffi-bindgen - command: make bindgen-python - run: name: Python lints command: make lint-python @@ -885,7 +882,7 @@ jobs: # Upload to GitHub ./ghr -replace ${CIRCLE_TAG} target/wheels - pypi-linux-aarch64-release: + pypi-linux-additional-releases: docker: - image: cimg/python:3.13 steps: @@ -898,19 +895,21 @@ jobs: make setup-python .venv3.13/bin/pip install ziglang - run: - name: Install aarch64-linux target + name: Install additional targets command: | rustup target add aarch64-unknown-linux-gnu + rustup target add aarch64-pc-windows-gnullvm - run: - name: Build Python package + name: Build Python packages command: | - # We need a binary with debug symbols, so uniffi-bindgen can extract data - cargo build -p glean-bundle - . .venv3.13/bin/activate - make build-python-wheel GLEAN_BUILD_TARGET=aarch64-unknown-linux-gnu GLEAN_BUILD_EXTRA="--zig" + + TARGETS="aarch64-unknown-linux-gnu aarch64-pc-windows-gnullvm" + for target in $TARGETS; do + make build-python-wheel GLEAN_BUILD_TARGET="$target" GLEAN_BUILD_EXTRA="--zig" + done - run: - name: Upload aarch64-linux wheel + name: Upload wheels command: | # Requires that the TWINE_USERNAME and TWINE_PASSWORD environment # variables are configured in CircleCI's environment variables. @@ -979,43 +978,6 @@ jobs: # Upload to GitHub ./ghr -replace ${CIRCLE_TAG} target/wheels - pypi-windows-aarch64-release: - docker: - - image: cimg/python:3.13 - steps: - - install-rustup - - setup-rust-toolchain - - checkout - - run: - name: Setup Python env - command: | - make setup-python - .venv3.13/bin/pip install ziglang - - run: - name: Install aarch64-windows target - command: | - rustup target add aarch64-pc-windows-gnullvm - - run: - name: Build Python package - command: | - # We need a binary with debug symbols, so uniffi-bindgen can extract data - cargo build -p glean-bundle - - . .venv3.13/bin/activate - make build-python-wheel GLEAN_BUILD_TARGET=aarch64-pc-windows-gnullvm GLEAN_BUILD_EXTRA="--zig" - - run: - name: Upload aarch64-windows wheel - command: | - # Requires that the TWINE_USERNAME and TWINE_PASSWORD environment - # variables are configured in CircleCI's environment variables. - make upload-wheels - - install-ghr-linux - - run: - name: Publish to GitHub - command: | - # Upload to GitHub - ./ghr -replace ${CIRCLE_TAG} target/wheels - ########################################################################### # Docs @@ -1215,7 +1177,7 @@ workflows: requires: - Python 3_9 tests filters: *release-filters - - pypi-linux-aarch64-release: + - pypi-linux-additional-releases: requires: - Python 3_9 tests filters: *release-filters @@ -1227,10 +1189,6 @@ workflows: requires: - Python 3_9 tests filters: *release-filters - - pypi-windows-aarch64-release: - requires: - - Python 3_9 tests - filters: *release-filters - iOS build and test: filters: *release-filters - iOS Framework release: diff --git a/Cargo.lock b/Cargo.lock index c8c227cce1..8f53b99cf7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1023,6 +1023,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c6d968cb62160c11f2573e6be724ef8b1b18a277aededd17033f8a912d73e2b4" dependencies = [ "anyhow", + "cargo_metadata", "uniffi_bindgen", "uniffi_build", "uniffi_core", @@ -1036,7 +1037,7 @@ version = "0.1.0" dependencies = [ "anyhow", "camino", - "glob", + "cargo_metadata", "uniffi", ] diff --git a/Cargo.toml b/Cargo.toml index 3b0875378c..03c1321b6a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,8 +17,9 @@ default-members = [ ] [profile.release] -opt-level = "s" -debug = false +opt-level = 2 +debug = "line-tables-only" +split-debuginfo = "unpacked" lto = true [profile.profiling] diff --git a/Makefile b/Makefile index 92352264e3..386ab200e4 100644 --- a/Makefile +++ b/Makefile @@ -57,16 +57,7 @@ build-python-sdist: setup-python ## Build a Python source distribution build-xcframework: ./bin/build-xcframework.sh -bindgen-python: glean-core/python/glean/_uniffi/glean.py glean-core/python/glean/_uniffi/__init__.py # Generate the uniffi wrapper code manually - -glean-core/python/glean/_uniffi/glean.py: glean-core/src/glean.udl - cargo build -p glean-bundle - cargo uniffi-bindgen generate $< --language python --out-dir $(@D) - -glean-core/python/glean/_uniffi/__init__.py: - echo 'from .glean import * # NOQA' > $@ - -.PHONY: build build-rust build-kotlin build-swift build-apk build-python build-python-wheel build-python-sdist bindgen-python build-xcframework glean-core/python/glean/_uniffi/__init__.py +.PHONY: build build-rust build-kotlin build-swift build-apk build-python build-python-wheel build-python-sdist build-xcframework glean-core/python/glean/_uniffi/__init__.py # All tests @@ -125,7 +116,7 @@ shellcheck: ## Run shellcheck against important shell scripts shellcheck glean-core/ios/sdk_generator.sh shellcheck bin/check-artifact.sh -lint-python: setup-python ## Run ruff and mypy to lint Python code +lint-python: build-python ## Run ruff and mypy to lint Python code $(GLEAN_PYENV)/bin/python3 -m ruff format --diff glean-core/python/glean glean-core/python/tests $(GLEAN_PYENV)/bin/python3 -m ruff check glean-core/python/glean glean-core/python/tests $(GLEAN_PYENV)/bin/python3 -m mypy glean-core/python/glean diff --git a/build-scripts/xc-bindgen.sh b/build-scripts/xc-bindgen.sh new file mode 100644 index 0000000000..90a236bd52 --- /dev/null +++ b/build-scripts/xc-bindgen.sh @@ -0,0 +1,58 @@ +#!/bin/bash + +# what to pass to cargo build -p, e.g. glean_ffi +FFI_TARGET=$1 +# path to Glean root +GLEAN_ROOT=$2 +# buildvariant from our xcconfigs +BUILDVARIANT=$3 + +RELMODE=debug +if [[ "$BUILDVARIANT" != "debug" ]]; then + RELMODE=release +fi + +set -euvx + +IS_SIMULATOR=0 +if [ "${LLVM_TARGET_TRIPLE_SUFFIX-}" = "-simulator" ]; then + IS_SIMULATOR=1 +fi + +TARGET= + +for arch in $ARCHS; do + case "$arch" in + x86_64) + # Intel iOS simulator + TARGET="x86_64-apple-ios" + break + ;; + + arm64) + if [ $IS_SIMULATOR -eq 0 ]; then + # Hardware iOS targets + TARGET=aarch64-apple-ios + else + # M1 iOS simulator + TARGET=aarch64-apple-ios-sim + fi + break + ;; + esac +done + +if [[ -z "$TARGET" ]]; then + echo "Missing TARGET. No suitable arch in ${ARCHS}" + exit 2 +fi + +LIBRARY_PATH="${GLEAN_ROOT}/target/${TARGET}/${RELMODE}/lib${FFI_TARGET}.a" + +bash "${GLEAN_ROOT}/build-scripts/xc-cargo.sh" cargo run --package uniffi-bindgen -- \ + generate \ + --language swift \ + --config "${GLEAN_ROOT}/../bundle/uniffi.toml" \ + --out-dir "${GLEAN_ROOT}/glean-core/ios/Glean/Generated/uniffi" \ + --library "$LIBRARY_PATH" \ + --no-format diff --git a/build-scripts/xc-universal-binary.sh b/build-scripts/xc-universal-binary.sh index f3cdb4355e..ded2230176 100644 --- a/build-scripts/xc-universal-binary.sh +++ b/build-scripts/xc-universal-binary.sh @@ -11,7 +11,7 @@ fi # what to pass to cargo build -p, e.g. glean_ffi FFI_TARGET=$1 -# path to app services root +# path to Glean root GLEAN_ROOT=$2 # buildvariant from our xcconfigs BUILDVARIANT=$3 diff --git a/glean-core/android/build.gradle b/glean-core/android/build.gradle index d2d114a199..b1d4b8cdd8 100644 --- a/glean-core/android/build.gradle +++ b/glean-core/android/build.gradle @@ -195,14 +195,25 @@ apply from: "$rootDir/publish.gradle" ext.configurePublish() def generateUniffiBindings = tasks.register("generateUniffiBindings") { - def udlFilePath = "../src/glean.udl" + def gleanNative = configurations.getByName("gleanNative") doFirst { + def libraryPath = gleanNative.asFileTree.matching { + include "${nativeRustTarget}/libxul.*" + }.singleFile + + if (libraryPath == null) { + throw new GradleException("libglean_ffi dynamic library path not found") + } + exec { workingDir project.rootDir - commandLine 'cargo', 'uniffi-bindgen', 'generate', '--no-format', "${project.projectDir}/${udlFilePath}", '--language', 'kotlin', '--out-dir', UNIFFI_OUT_DIR.get() + commandLine 'cargo', 'uniffi-bindgen', 'generate', '--no-format', '--library', libraryPath, '--language', 'kotlin', '--out-dir', UNIFFI_OUT_DIR.get() } } + outputs.dir UNIFFI_OUT_DIR + // Re-generate if native changes + inputs.files gleanNative // Re-generate if the interface definition changes. inputs.file "${project.projectDir}/../src/glean.udl" // Re-generate if the uniffi config changes. diff --git a/glean-core/ios/Glean.xcodeproj/project.pbxproj b/glean-core/ios/Glean.xcodeproj/project.pbxproj index 85547eba02..57c9ac823b 100644 --- a/glean-core/ios/Glean.xcodeproj/project.pbxproj +++ b/glean-core/ios/Glean.xcodeproj/project.pbxproj @@ -588,6 +588,7 @@ ); inputPaths = ( "$(SRCROOT)/../src/glean.udl", + "$(SRCROOT)/../../build-scripts/xc-cargo.sh", ); name = "Run UniFFI bindgen"; outputFileListPaths = ( @@ -598,7 +599,7 @@ ); runOnlyForDeploymentPostprocessing = 0; shellPath = /bin/sh; - shellScript = "if [ \"$ACTION\" = \"indexbuild\" ]; then\n echo \"Skipping code generation in 'indexbuild' build. See https://bugzilla.mozilla.org/show_bug.cgi?id=1744504 for more info.\"\n exit 0\nfi\nbash $PWD/../../build-scripts/xc-cargo.sh cargo run --package uniffi-bindgen -- generate $INPUT_FILE_PATH --language swift --out-dir $SRCROOT/Glean/Generated/uniffi --no-format $SCRIPT_INPUT_FILE_0\n"; + shellScript = "if [ \"$ACTION\" = \"indexbuild\" ]; then\n echo \"Skipping code generation in 'indexbuild' build. See https://bugzilla.mozilla.org/show_bug.cgi?id=1744504 for more info.\"\n exit 0\nfi\nbash $PWD/../../build-scripts/xc-bindgen.sh glean_ffi $PWD/../.. $buildvariant\n"; }; /* End PBXShellScriptBuildPhase section */ diff --git a/glean-core/python/requirements_dev.txt b/glean-core/python/requirements_dev.txt index a68cdbddd6..1d08b6827c 100644 --- a/glean-core/python/requirements_dev.txt +++ b/glean-core/python/requirements_dev.txt @@ -12,5 +12,5 @@ semver==2.13.0 setuptools-git==1.2 twine==6.0.1 wheel==0.45.1 -maturin==1.9.6 +maturin==1.10.1 patchelf>=0.17; sys_platform == "linux" diff --git a/tools/embedded-uniffi-bindgen/Cargo.toml b/tools/embedded-uniffi-bindgen/Cargo.toml index d5d262f192..9c58875226 100644 --- a/tools/embedded-uniffi-bindgen/Cargo.toml +++ b/tools/embedded-uniffi-bindgen/Cargo.toml @@ -8,8 +8,8 @@ publish = false [dependencies] anyhow = "1" camino = "1.1.1" -uniffi = { version = "0.29.3", default-features = false, features = ["bindgen"] } -glob = "0.3.2" +uniffi = { version = "0.29.3", default-features = false, features = ["bindgen", "cargo-metadata"] } +cargo_metadata = { version = "0.19" } [lib] test = false diff --git a/tools/embedded-uniffi-bindgen/src/main.rs b/tools/embedded-uniffi-bindgen/src/main.rs index a0cd8b13cb..dce42852c7 100644 --- a/tools/embedded-uniffi-bindgen/src/main.rs +++ b/tools/embedded-uniffi-bindgen/src/main.rs @@ -6,7 +6,6 @@ use std::env; use anyhow::{bail, Context}; use camino::Utf8PathBuf; -use glob::glob; #[derive(Copy, Clone, Eq, PartialEq, Hash)] enum TargetLanguage { @@ -24,92 +23,56 @@ fn parse_language(lang: &str) -> anyhow::Result { } } -fn find_library_file(crate_root: &camino::Utf8Path, library_name: &str) -> Utf8PathBuf { - let path = if let Ok(target_dir) = env::var("CARGO_TARGET_DIR") { - Utf8PathBuf::from(target_dir) - } else { - crate_root.join("target") - }; - - // Search for all viable patterns and pick the first one we can find. - let glob_patterns = [ - format!("{path}/**/lib{library_name}.a"), - format!("{path}/**/lib{library_name}.so"), - format!("{path}/**/lib{library_name}.dylib"), - format!("{path}/**/{library_name}.dll"), - ]; - - for pattern in &glob_patterns { - if let Some(Ok(path)) = glob(pattern).unwrap().next() { - return Utf8PathBuf::from_path_buf(path).unwrap(); - } - } - - panic!("lib{library_name} could not be found in {path}") -} - fn gen_bindings( - udl_file: &camino::Utf8Path, + library_file: &camino::Utf8Path, config_file: Option<&camino::Utf8Path>, languages: Vec, - out_dir: Option<&camino::Utf8Path>, - crate_name: Option<&str>, + out_dir: &camino::Utf8Path, + crate_name: Option, ) -> anyhow::Result<()> { - use uniffi::generate_bindings; + use uniffi::generate_bindings_library_mode; use uniffi::{KotlinBindingGenerator, PythonBindingGenerator, SwiftBindingGenerator}; - // 3 parents - // Should be `glean-core/src/glean.udl`, - // the one in `glean-core/bundle` is a symlink. - let canonical = udl_file.canonicalize().unwrap(); - let crate_root = camino::Utf8Path::from_path( - canonical - .parent() - .unwrap() - .parent() - .unwrap() - .parent() - .unwrap(), - ) - .unwrap(); + use uniffi::CargoMetadataConfigSupplier; + let mut cmd = cargo_metadata::MetadataCommand::new(); + cmd.no_deps(); + let metadata = cmd.exec().context("error running cargo metadata")?; + let config_supplier = CargoMetadataConfigSupplier::from(metadata); for language in languages { match language { TargetLanguage::Kotlin => { - let library_file = find_library_file(crate_root, "xul"); - generate_bindings( - udl_file, + generate_bindings_library_mode( + library_file, + crate_name.clone(), + &KotlinBindingGenerator, + &config_supplier, config_file, - KotlinBindingGenerator, out_dir, - Some(&library_file), - crate_name, false, - )? + )?; } TargetLanguage::Python => { - let library_file = find_library_file(crate_root, "glean_ffi"); - generate_bindings( - udl_file, + generate_bindings_library_mode( + library_file, + crate_name.clone(), + &PythonBindingGenerator, + &config_supplier, config_file, - PythonBindingGenerator, out_dir, - Some(&library_file), - crate_name, false, - )? + )?; } TargetLanguage::Swift => { - let library_file = find_library_file(crate_root, "glean_ffi"); - generate_bindings( - udl_file, + generate_bindings_library_mode( + library_file, + crate_name.clone(), + &SwiftBindingGenerator, + &config_supplier, config_file, - SwiftBindingGenerator, out_dir, - Some(&library_file), - crate_name, false, - )? + )?; } }; } @@ -123,7 +86,7 @@ fn main() -> anyhow::Result<()> { bail!("Only the `generate` subcommand is supported."); } - let mut udl_file = None; + let mut library_file = None; let mut target_languages = vec![]; let mut out_dir = None; let mut config = None; @@ -143,20 +106,22 @@ fn main() -> anyhow::Result<()> { "config" => { config = Some(args.next().context("--config needs a parameter")?); } + "library" => { + library_file = Some(args.next().context("--library needs a parameter")?); + } _ => bail!("Unsupported option: {arg}"), } - } else if udl_file.is_some() { - bail!("UDL file already set."); } else { - udl_file = Some(Utf8PathBuf::from(arg)); + bail!("Unknown parameter {arg:?}"); } } + let library_file = library_file.map(Utf8PathBuf::from); let out_dir = out_dir.map(Utf8PathBuf::from); let config = config.map(Utf8PathBuf::from); - if udl_file.is_none() { - bail!("Need UDL file."); + if library_file.is_none() { + bail!("Need path to library file."); } if target_languages.is_empty() { @@ -168,11 +133,11 @@ fn main() -> anyhow::Result<()> { } gen_bindings( - &udl_file.unwrap(), + &library_file.unwrap(), config.as_deref(), target_languages, - out_dir.as_deref(), - Some("glean_core"), + &out_dir.unwrap(), + Some(String::from("glean_core")), )?; Ok(())