diff --git a/plugins/pyenv-binary/libexec/pyenv-binary-generate-installer b/plugins/pyenv-binary/libexec/pyenv-binary-generate-installer index 39ca7052..58e5d461 100755 --- a/plugins/pyenv-binary/libexec/pyenv-binary-generate-installer +++ b/plugins/pyenv-binary/libexec/pyenv-binary-generate-installer @@ -55,7 +55,7 @@ if [ -z "$metadata" ] || [ -z "$archive_url" ]; then exit 1 fi -version="" os="" arch="" distro="" libc="" archive="" +version="" os="" arch="" distro="" libc="" build_prefix="" archive="" deps="" while IFS='=' read -r key value; do case "$key" in @@ -64,6 +64,7 @@ while IFS='=' read -r key value; do arch ) arch="$value" ;; distro ) distro="$value" ;; libc ) libc="$value" ;; + build_prefix ) build_prefix="$value" ;; archive ) archive="$value" ;; dep ) deps="${deps:+$deps }$value" ;; esac @@ -78,10 +79,9 @@ for field in version os arch archive; do fi done -# macOS relocation needs install_name_tool and a different rpath scheme, so it is -# not supported yet. Other systems relocate with patchelf. -if [ "$os" = "Darwin" ]; then - echo "pyenv-binary: macOS archives are not supported yet" >&2 +# Mach-O load commands contain the prefix where Python was built. +if [ "$os" = "Darwin" ] && [ -z "$build_prefix" ]; then + echo "pyenv-binary: metadata is missing \`build_prefix'" >&2 exit 1 fi @@ -122,6 +122,7 @@ emit() { printf 'EXPECT_OS=%q\n' "$os" printf 'EXPECT_ARCH=%q\n' "$arch" printf 'EXPECT_LIBC=%q\n' "$libc" + printf 'BUILD_PREFIX=%q\n' "$build_prefix" printf 'VERSION=%q\n' "$version" printf 'ARCHIVE_URL=%q\n' "$archive_url" printf 'SHA256=%q\n' "$sha256" @@ -185,13 +186,29 @@ elif [ -n "$DEPS" ]; then echo "pyenv-binary: cannot check required system libraries (ldconfig cache unavailable)" >&2 fi -if ! command -v patchelf >/dev/null 2>&1; then - echo "pyenv-binary: need patchelf to relocate the binary" >&2 - exit 1 -fi +case "$os" in +Darwin ) + for tool in otool install_name_tool; do + if ! command -v "$tool" >/dev/null 2>&1; then + echo "pyenv-binary: need ${tool} to relocate the binary" >&2 + exit 1 + fi + done + ;; +* ) + if ! command -v patchelf >/dev/null 2>&1; then + echo "pyenv-binary: need patchelf to relocate the binary" >&2 + exit 1 + fi + ;; +esac build_package_relocate() { - pyenv binary relocate "$PREFIX_PATH" + if [ "$os" = "Darwin" ]; then + pyenv binary relocate "$PREFIX_PATH" "$BUILD_PREFIX" + else + pyenv binary relocate "$PREFIX_PATH" + fi } install_package "Python-${VERSION}-binary" "${ARCHIVE_URL}#${SHA256}" copy relocate diff --git a/plugins/pyenv-binary/libexec/pyenv-binary-package b/plugins/pyenv-binary/libexec/pyenv-binary-package index 4c6592bc..f86666e2 100755 --- a/plugins/pyenv-binary/libexec/pyenv-binary-package +++ b/plugins/pyenv-binary/libexec/pyenv-binary-package @@ -75,15 +75,6 @@ latest ) ;; esac -os="$(uname -s)" - -# `generate-installer' refuses a macOS archive, so the last step here cannot -# succeed on one. Give up before compiling rather than after it. -if [ "$os" = "Darwin" ]; then - echo "pyenv-binary: macOS archives are not supported yet" >&2 - exit 1 -fi - # `pyenv install' puts a `:' build under versions/. pyenv-install ${verbose:+--verbose} "$spec" pyenv-binary-save "$entry" "$PWD" --name "$entry" diff --git a/plugins/pyenv-binary/libexec/pyenv-binary-relocate b/plugins/pyenv-binary/libexec/pyenv-binary-relocate index 6b977893..964ab543 100755 --- a/plugins/pyenv-binary/libexec/pyenv-binary-relocate +++ b/plugins/pyenv-binary/libexec/pyenv-binary-relocate @@ -1,12 +1,13 @@ #!/usr/bin/env bash # -# Summary: Rewrite an unpacked Python's rpaths so it runs from its prefix +# Summary: Rewrite an unpacked Python's library paths so it runs from its prefix # -# Usage: pyenv binary relocate +# Usage: pyenv binary relocate [] # -# Rewrites the rpaths of a Python tree that was unpacked into so the -# interpreter and its extension modules load the bundled libraries from their -# new location rather than the path it was built at. Requires patchelf. +# Rewrites the library paths of a Python tree that was unpacked into +# so the interpreter and its extension modules load the bundled libraries from +# their new location rather than the path it was built at. Requires patchelf on +# Linux and FreeBSD, or otool and install_name_tool on macOS. # # The definition that `pyenv binary generate-installer' produces calls this # after `install_package ... copy' has laid the tree down. @@ -24,6 +25,84 @@ if [ -z "$prefix" ]; then exit 1 fi +if [ "$(uname -s)" = "Darwin" ]; then + build_prefix="${2%/}" + if [ -z "$build_prefix" ]; then + echo "pyenv-binary: need the original build prefix to relocate a macOS binary" >&2 + exit 1 + fi + for tool in otool install_name_tool; do + if ! command -v "$tool" >/dev/null 2>&1; then + echo "pyenv-binary: need ${tool} to relocate the binary" >&2 + exit 1 + fi + done + + relocate_macho() { + local file="$1" new_rpath="$2" + local load_commands install_ids rpaths dependency install_id rpath replacement + + load_commands="$(otool -L "$file")" + install_ids="$(otool -D "$file")" + rpaths="$(otool -l "$file")" + install_id="$(printf '%s\n' "$install_ids" | sed -n '2s/^[[:space:]]*//p')" + + while IFS= read -r dependency; do + case "$dependency" in + "$build_prefix"/lib/* ) + if [ "$dependency" != "$install_id" ]; then + replacement="@rpath/${dependency#"$build_prefix"/lib/}" + install_name_tool -change "$dependency" "$replacement" "$file" + fi + ;; + esac + done < <(printf '%s\n' "$load_commands" | tail -n +2 | sed -E \ + 's/^[[:space:]]*//; s/[[:space:]]+\(compatibility version.*$//') + + case "$install_id" in + "$build_prefix"/lib/* ) + replacement="@rpath/${install_id#"$build_prefix"/lib/}" + install_name_tool -id "$replacement" "$file" + ;; + esac + + while IFS= read -r rpath; do + if [ "$rpath" = "$build_prefix/lib" ]; then + install_name_tool -rpath "$rpath" "$new_rpath" "$file" + fi + done < <(printf '%s\n' "$rpaths" | awk ' + /cmd LC_RPATH/ { found = 1; next } + found && /^[[:space:]]*path / { + sub(/^[[:space:]]*path /, "") + sub(/ \(offset [0-9]+\)$/, "") + print + found = 0 + } + ') + } + + found=0 + for file in "$prefix"/bin/*; do + [[ -f $file && -x $file && ! -L $file ]] || continue + otool -L "$file" >/dev/null 2>&1 || continue + relocate_macho "$file" '@executable_path/../lib' + found=1 + done + if [ "$found" -eq 0 ]; then + echo "pyenv-binary: found no interpreter to relocate under \`${prefix}/bin'" >&2 + exit 1 + fi + + for file in "$prefix"/lib/*.dylib; do + [[ -f $file && ! -L $file ]] || continue + relocate_macho "$file" '@loader_path' + done + while IFS= read -r so; do + relocate_macho "$so" '@loader_path/../..' + done < <(find "$prefix" -type f -path '*/lib/python*/lib-dynload/*.so') + exit +fi + # patchelf can add an rpath and write one of any length; chrpath can only shorten # an existing one, which is not enough to relocate every build, so require it. if ! command -v patchelf >/dev/null 2>&1; then diff --git a/plugins/pyenv-binary/libexec/pyenv-binary-save b/plugins/pyenv-binary/libexec/pyenv-binary-save index 8e2405b2..b940123a 100755 --- a/plugins/pyenv-binary/libexec/pyenv-binary-save +++ b/plugins/pyenv-binary/libexec/pyenv-binary-save @@ -147,6 +147,7 @@ tar -C "$(dirname "$prefix")" -czf "${output_dir}/${archive}" "$(basename "$pref echo "platform=${platform}" [ -n "$distro" ] && echo "distro=${distro}" [ -n "$libc" ] && echo "libc=${libc}" + echo "build_prefix=${prefix}" echo "archive=${archive}" system_deps | while IFS= read -r dep; do [ -n "$dep" ] && echo "dep=${dep}" diff --git a/plugins/pyenv-binary/test/generate-installer.bats b/plugins/pyenv-binary/test/generate-installer.bats index 02f63fac..74fdfe69 100644 --- a/plugins/pyenv-binary/test/generate-installer.bats +++ b/plugins/pyenv-binary/test/generate-installer.bats @@ -6,6 +6,7 @@ create_meta() { local os="${1-Linux}" local arch="${2-x86_64}" local libc="${3-glibc 2.17}" + local build_prefix="${4-}" local archive="${BATS_TEST_TMPDIR}/3.12.7.tar.gz" local meta="${BATS_TEST_TMPDIR}/sample.meta" @@ -20,6 +21,7 @@ create_meta() { echo "os=${os}" echo "arch=${arch}" [ -z "$libc" ] || echo "libc=${libc}" + [ -z "$build_prefix" ] || echo "build_prefix=${build_prefix}" echo "archive=${archive##*/}" } > "$meta" echo "$meta" @@ -58,10 +60,20 @@ create_meta() { assert_failure } -@test "refuses macOS metadata" { +@test "fails when macOS metadata is missing the build prefix" { run pyenv-binary-generate-installer "$(create_meta Darwin arm64 '')" \ --archive-url http://x/a.tar.gz - assert_failure "pyenv-binary: macOS archives are not supported yet" + assert_failure "pyenv-binary: metadata is missing \`build_prefix'" +} + +@test "quotes the macOS build prefix in the generated definition" { + local out="${BATS_TEST_TMPDIR}/definition" + pyenv-binary-generate-installer \ + "$(create_meta Darwin arm64 '' '/build prefix/$name')" \ + --archive-url http://example.com/a.tar.gz -o "$out" + + run grep '^BUILD_PREFIX=' "$out" + assert_success 'BUILD_PREFIX=/build\ prefix/\$name' } @test "fails when required metadata is missing" { @@ -117,6 +129,56 @@ create_meta() { assert_failure "pyenv-binary: need patchelf to relocate the binary" } +@test "checks for otool before installing a macOS archive" { + local out="${BATS_TEST_TMPDIR}/definition" + pyenv-binary-generate-installer \ + "$(create_meta Darwin arm64 '' /build/3.12.7)" \ + --archive-url http://example.com/a.tar.gz -o "$out" + create_stub uname 'case "$1" in -s) echo Darwin;; -m) echo arm64;; esac' + create_path_executable install_name_tool true + + PATH="${BATS_TEST_TMPDIR}/stubs:${PYENV_TEST_DIR}/bin:/bin" run bash "$out" + assert_failure "pyenv-binary: need otool to relocate the binary" +} + +@test "checks for install_name_tool before installing a macOS archive" { + local out="${BATS_TEST_TMPDIR}/definition" + pyenv-binary-generate-installer \ + "$(create_meta Darwin arm64 '' /build/3.12.7)" \ + --archive-url http://example.com/a.tar.gz -o "$out" + create_stub uname 'case "$1" in -s) echo Darwin;; -m) echo arm64;; esac' + create_path_executable otool true + + PATH="${BATS_TEST_TMPDIR}/stubs:${PYENV_TEST_DIR}/bin:/bin" run bash "$out" + assert_failure "pyenv-binary: need install_name_tool to relocate the binary" +} + +@test "the generated macOS definition preserves both relocation arguments" { + local archive="${BATS_TEST_TMPDIR}/3.12.7.tar.gz" + local cache="${BATS_TEST_TMPDIR}/cache" + local definition="${BATS_TEST_TMPDIR}/definition" + local prefix="${BATS_TEST_TMPDIR}/install" + pyenv-binary-generate-installer \ + "$(create_meta Darwin arm64 '' '/build prefix/$name')" \ + --archive-url http://example.com/3.12.7.tar.gz -o "$definition" + mkdir -p "$cache" + cp "$archive" "${cache}/Python-3.12.7-binary.tar.gz" + create_stub pyenv 'printf "argc=%s\narg1=%s\narg2=%s\narg3=%s\narg4=%s\n" "$#" "$1" "$2" "$3" "$4"' + create_stub uname 'case "$1" in -s) echo Darwin;; -m) echo arm64;; esac' + create_path_executable otool true + create_path_executable install_name_tool true + + PYTHON_BUILD_CACHE_PATH="$cache" run \ + "${BATS_TEST_DIRNAME}/../../python-build/bin/python-build" "$definition" "$prefix" + assert_success + assert_line "argc=4" + assert_line "arg1=binary" + assert_line "arg2=relocate" + assert_line "arg3=${prefix}" + assert_line 'arg4=/build prefix/$name' + assert_line "Installed Python-3.12.7-binary to ${prefix}" +} + @test "fails when the archive is not beside the metadata" { local meta="$(create_meta)" rm "${BATS_TEST_TMPDIR}/3.12.7.tar.gz" diff --git a/plugins/pyenv-binary/test/package.bats b/plugins/pyenv-binary/test/package.bats index 51da03a2..4186a1c8 100644 --- a/plugins/pyenv-binary/test/package.bats +++ b/plugins/pyenv-binary/test/package.bats @@ -82,10 +82,16 @@ pyenv-install --list --bare assert_failure "pyenv-binary: \`latest' cannot be used as an entry name" } -@test "refuses to package on macOS before compiling anything" { +@test "packages on macOS" { create_stub uname 'case "$1" in -s) echo Darwin;; -m) echo arm64;; esac' + create_stub pyenv-install 'echo install' + create_stub pyenv-binary-save 'echo save' + create_stub pyenv-binary-generate-installer 'echo generate-installer' + run pyenv-binary-package 3.12.7:3.12.7-test --archive-base-url http://x/b - assert_failure "pyenv-binary: macOS archives are not supported yet" + assert_success "install +save +generate-installer" } @test "writes the archive, metadata and definition under the entry name (integration)" { diff --git a/plugins/pyenv-binary/test/relocate.bats b/plugins/pyenv-binary/test/relocate.bats index 0051a42c..bfac5d39 100644 --- a/plugins/pyenv-binary/test/relocate.bats +++ b/plugins/pyenv-binary/test/relocate.bats @@ -4,6 +4,7 @@ load test_helper _setup() { create_stub pyenv-help "echo usage" + create_stub uname 'case "$1" in -s) echo Linux;; -m) echo x86_64;; esac' } stub_patchelf() { @@ -21,6 +22,84 @@ create_interpreter() { chmod +x "${BATS_TEST_TMPDIR}/prefix/bin/python2.7" } +create_macos_tree() { + local prefix="${BATS_TEST_TMPDIR}/prefix" + local lib="${prefix}/lib/python3.12" + mkdir -p "${prefix}/bin" "${lib}/lib-dynload" "${lib}/site-packages/numpy" + printf '#!/bin/sh\n' > "${prefix}/bin/python3.12" + chmod +x "${prefix}/bin/python3.12" + touch "${prefix}/lib/libpython3.12.dylib" + touch "${lib}/lib-dynload/_ssl.cpython-312-darwin.so" + touch "${lib}/site-packages/numpy/_multiarray.so" +} + +stub_macos_tools() { + create_stub uname 'case "$1" in -s) echo Darwin;; -m) echo arm64;; esac' + create_path_executable install_name_tool \ + 'echo "$*" >> "${BATS_TEST_TMPDIR}/install-name-tool.log"' + create_path_executable otool <<'STUB' +file="${!#}" +echo "$*" >> "${BATS_TEST_TMPDIR}/otool.log" +old="/build prefix/3.12.7" +case "$1" in +-L ) + echo "${file}:" + case "$file" in + */bin/python3.12 ) + if [ -n "$OTOOL_RELOCATED" ]; then + echo " @rpath/libpython3.12.dylib (compatibility version 3.12.0, current version 3.12.0)" + else + echo " ${old}/lib/libpython3.12.dylib (compatibility version 3.12.0, current version 3.12.0)" + fi + echo " ${old}-other/lib/libother.dylib (compatibility version 1.0.0, current version 1.0.0)" + echo " /usr/lib/libSystem.B.dylib (compatibility version 1.0.0, current version 1.0.0)" + echo " /opt/homebrew/lib/libintl.8.dylib (compatibility version 1.0.0, current version 1.0.0)" + echo " @loader_path/liblocal.dylib (compatibility version 1.0.0, current version 1.0.0)" + ;; + */lib/libpython3.12.dylib ) + if [ -n "$OTOOL_RELOCATED" ]; then + echo " @rpath/libpython3.12.dylib (compatibility version 3.12.0, current version 3.12.0)" + else + echo " ${old}/lib/libpython3.12.dylib (compatibility version 3.12.0, current version 3.12.0)" + fi + ;; + esac + ;; +-D ) + echo "${file}:" + case "$file" in + */lib/libpython3.12.dylib ) + if [ -n "$OTOOL_RELOCATED" ]; then + echo '@rpath/libpython3.12.dylib' + else + echo "${old}/lib/libpython3.12.dylib" + fi + ;; + esac + ;; +-l ) + if [ -n "$OTOOL_RELOCATED" ]; then + case "$file" in + */bin/* ) rpath='@executable_path/../lib' ;; + */lib/*.dylib ) rpath='@loader_path' ;; + * ) rpath='@loader_path/../..' ;; + esac + else + rpath="${old}/lib" + fi + cat </dev/null + + run grep '^build_prefix=' "${out}/3.12.7-$(platform).meta" + assert_success "build_prefix=${PYENV_ROOT}/versions/3.12.7" +} + @test "fails when readelf is not available" { create_version "3.12.7" create_stub uname 'case "$1" in -s) echo Linux;; -m) echo x86_64;; esac'