From 37ab83f1b0a5a01d92385e22948eaaa093b8ee00 Mon Sep 17 00:00:00 2001 From: Sam Doran Date: Wed, 29 Apr 2026 01:08:52 -0400 Subject: [PATCH] Improve performance and output of `pyenv virtualenvs` (#502) * With 160 envs, cuts time from ~14s to ~2.5s * Change the output to be more like `pyenv-versions' * Shellcheck fixes * virtualenv: Only show non-envs for completion since env engines don't support piggy-backing envs * Remove dead/unused code * Fix IFS handling `IFS=: ' doesn't apply that IFS to expansions in , it has to be set separately earlier * Adjust and refactor tests --------- Co-authored-by: Ivan Pozdeev --- bin/pyenv-virtualenv | 2 +- bin/pyenv-virtualenv-prefix | 28 ++++-- bin/pyenv-virtualenvs | 129 ++++++++++++++------------ test/conda-prefix.bats | 4 - test/prefix.bats | 68 ++++---------- test/virtualenvs.bats | 174 ++++++++++++++++++++++++++++++------ 6 files changed, 257 insertions(+), 148 deletions(-) diff --git a/bin/pyenv-virtualenv b/bin/pyenv-virtualenv index f1cbe3d..0619f62 100755 --- a/bin/pyenv-virtualenv +++ b/bin/pyenv-virtualenv @@ -24,7 +24,7 @@ fi # Provide pyenv completions if [ "$1" = "--complete" ]; then - exec pyenv-versions --bare + exec pyenv-versions --bare --skip-envs fi unset PIP_REQUIRE_VENV diff --git a/bin/pyenv-virtualenv-prefix b/bin/pyenv-virtualenv-prefix index bbe5871..198096a 100755 --- a/bin/pyenv-virtualenv-prefix +++ b/bin/pyenv-virtualenv-prefix @@ -26,13 +26,17 @@ if [ -z "$PYENV_ROOT" ]; then PYENV_ROOT="${HOME}/.pyenv" fi +OLDIFS="$IFS" +IFS=: if [ -n "$1" ]; then - versions=($@) - IFS=: PYENV_VERSION="${versions[*]}" + # $@ is not affected by IFS + versions=("$@") + PYENV_VERSION="${versions[*]}" export PYENV_VERSION else - IFS=: versions=($(pyenv-version-name)) + versions=($(pyenv-version-name)) fi +IFS="$OLDIFS" append_virtualenv_prefix() { if [ -d "${VIRTUALENV_PREFIX_PATH}" ]; then @@ -49,7 +53,18 @@ for version in "${versions[@]}"; do echo "pyenv-virtualenv: version \`${version}' is not a virtualenv" 1>&2 exit 1 fi - PYENV_PREFIX_PATH="$(pyenv-prefix "${version}")" + + # In the vast majority of cases, there's a direct match and + # not spawning `pyenv-prefix' saves about half the invocation time + # with a signle argument which accumulates when called repeatedly + # (e.g. from `pyenv-virtualenvs'). + # `pyenv-prefix' also does not have hooks to worry about. + # XXX: refactor the test into a shared module? + PYENV_PREFIX_PATH="${PYENV_ROOT}/versions/${version}" + if [[ ! -d "$PYENV_PREFIX_PATH" ]]; then + PYENV_PREFIX_PATH="$(pyenv-prefix "${version}")" + fi + if [ -x "${PYENV_PREFIX_PATH}/bin/python" ]; then if [ -f "${PYENV_PREFIX_PATH}/bin/activate" ]; then if [ -f "${PYENV_PREFIX_PATH}/bin/conda" ]; then @@ -94,4 +109,7 @@ for version in "${versions[@]}"; do fi done -IFS=: echo "${VIRTUALENV_PREFIX_PATHS[*]}" +OLDIFS="$IFS" +IFS=: +echo "${VIRTUALENV_PREFIX_PATHS[*]}" +IFS="$OLDIFS" diff --git a/bin/pyenv-virtualenvs b/bin/pyenv-virtualenvs index fec1286..8676574 100755 --- a/bin/pyenv-virtualenvs +++ b/bin/pyenv-virtualenvs @@ -6,24 +6,9 @@ # List all virtualenvs found in `$PYENV_ROOT/versions/*' and its `$PYENV_ROOT/versions/envs/*'. set -e -[ -n "$PYENV_DEBUG" ] && set -x -if [ -L "${BASH_SOURCE}" ]; then - READLINK=$(type -p greadlink readlink | head -1) - if [ -z "$READLINK" ]; then - echo "pyenv: cannot find readlink - are you missing GNU coreutils?" >&2 - exit 1 - fi - resolve_link() { - $READLINK -f "$1" - } - script_path=$(resolve_link ${BASH_SOURCE}) -else - script_path=${BASH_SOURCE} -fi +[[ -n $PYENV_DEBUG ]] && set -x -. ${script_path%/*}/../libexec/pyenv-virtualenv-realpath - -if [ -z "$PYENV_ROOT" ]; then +if [[ -z $PYENV_ROOT ]]; then PYENV_ROOT="${HOME}/.pyenv" fi @@ -47,28 +32,26 @@ done versions_dir="${PYENV_ROOT}/versions" -if [ -d "$versions_dir" ]; then - versions_dir="$(realpath "$versions_dir")" -fi - -if [ -n "$bare" ]; then - hit_prefix="" - miss_prefix="" - current_versions=() - unset print_origin - include_system="" +if [[ ${BASH_VERSINFO[0]} -gt 3 ]]; then + declare -A current_versions else + current_versions=() +fi +if [[ -z $bare ]]; then hit_prefix="* " miss_prefix=" " OLDIFS="$IFS" - IFS=: current_versions=($(pyenv-version-name || true)) + IFS=: + if [[ ${BASH_VERSINFO[0]} -gt 3 ]]; then + for i in $(pyenv-version-name || true); do + current_versions["$i"]="1" + done + else + read -r -a current_versions <<< "$(pyenv-version-name || true)" + fi IFS="$OLDIFS" - print_origin="1" - include_system="" fi -num_versions=0 - exists() { local car="$1" local cdar @@ -82,39 +65,67 @@ exists() { } print_version() { - if exists "$1" "${current_versions[@]}"; then - echo "${hit_prefix}${1}${print_origin+$2}" - else - echo "${miss_prefix}${1}${print_origin+$2}" + local version="${1:?}" + if [[ -n $bare ]]; then + echo "$version" + return + fi + local path="${2:?}" + if [[ -L "$path" ]]; then + version_repr="$version --> $(readlink "$path")" + else + version_repr="$version (created from $(pyenv-virtualenv-prefix "$version" 2>/dev/null))" + fi + if [[ ${BASH_VERSINFO[0]} -gt 3 && ${current_versions["$1"]} ]] || \ + { [[ ${BASH_VERSINFO[0]} -le 3 ]] && exists "$1" "${current_versions[@]}"; }; then + echo "${hit_prefix}${version_repr} (set by $(pyenv-version-origin))" + else + echo "${miss_prefix}${version_repr}" fi - num_versions=$((num_versions + 1)) } shopt -s dotglob shopt -s nullglob -for path in "$versions_dir"/*; do - if [ -d "$path" ]; then - if [ -n "$skip_aliases" ] && [ -L "$path" ]; then - target="$(realpath "$path")" - [ "${target%/*/envs/*}" != "$versions_dir" ] || continue - fi - virtualenv_prefix="$(pyenv-virtualenv-prefix "${path##*/}" 2>/dev/null || true)" - if [ -d "${virtualenv_prefix}" ]; then - print_version "${path##*/}" " (created from ${virtualenv_prefix})" - fi - for venv_path in "${path}/envs/"*; do - venv="${path##*/}/envs/${venv_path##*/}" - virtualenv_prefix="$(pyenv-virtualenv-prefix "${venv}" 2>/dev/null || true)" - if [ -d "${virtualenv_prefix}" ]; then - print_version "${venv}" " (created from ${virtualenv_prefix})" - fi - done +version_dir_entries=("$versions_dir"/*) +venv_dir_entries=("$versions_dir"/*/envs/*) + +if sort --version-sort /dev/null 2>&1; then + # system sort supports version sorting + OLDIFS="$IFS" + IFS='||' + + read -r -a version_dir_entries <<< "$( + printf "%s||" "${version_dir_entries[@]}" | + sort --version-sort + )" + + read -r -a venv_dir_entries <<< "$( + printf "%s||" "${venv_dir_entries[@]}" | + sort --version-sort + )" + + IFS="$OLDIFS" +fi + +for env_path in "${venv_dir_entries[@]}"; do + if [[ -d ${env_path} ]]; then + print_version "${env_path#"${PYENV_ROOT}"/versions/}" "${env_path}" fi done + +for env_path in "${version_dir_entries[@]}"; do + if [[ -d ${env_path} ]]; then + if [[ -L ${env_path} ]]; then + if [[ -z $skip_aliases ]]; then + print_version "${env_path#"${PYENV_ROOT}"/versions/}" "${env_path}" + fi + # Mimics the test from pyenv-virtualenv-prefix + # XXX: refactor itto a shared module ? + elif [[ -f "${env_path}/bin/activate" ]]; then + print_version "${env_path#"${PYENV_ROOT}"/versions/}" "${env_path}" + fi + fi +done + shopt -u dotglob shopt -u nullglob - -if [ "$num_versions" -eq 0 ] && [ -n "$include_system" ]; then - echo "Warning: no Python virtualenv detected on the system" >&2 - exit 1 -fi diff --git a/test/conda-prefix.bats b/test/conda-prefix.bats index 6965d52..4c2436a 100644 --- a/test/conda-prefix.bats +++ b/test/conda-prefix.bats @@ -9,7 +9,6 @@ setup() { @test "display conda root" { setup_conda "anaconda-2.3.0" stub pyenv-version-name "echo anaconda-2.3.0" - stub pyenv-prefix "anaconda-2.3.0 : echo \"${PYENV_ROOT}/versions/anaconda-2.3.0\"" PYENV_VERSION="anaconda-2.3.0" run pyenv-virtualenv-prefix @@ -19,14 +18,12 @@ ${PYENV_ROOT}/versions/anaconda-2.3.0 OUT unstub pyenv-version-name - unstub pyenv-prefix teardown_conda "anaconda-2.3.0" } @test "display conda env" { setup_conda "anaconda-2.3.0" "foo" stub pyenv-version-name "echo anaconda-2.3.0/envs/foo" - stub pyenv-prefix "anaconda-2.3.0/envs/foo : echo \"${PYENV_ROOT}/versions/anaconda-2.3.0/envs/foo\"" PYENV_VERSION="anaconda-2.3.0/envs/foo" run pyenv-virtualenv-prefix @@ -36,6 +33,5 @@ ${PYENV_ROOT}/versions/anaconda-2.3.0/envs/foo OUT unstub pyenv-version-name - unstub pyenv-prefix teardown_conda "anaconda-2.3.0" "foo" } diff --git a/test/prefix.bats b/test/prefix.bats index a21c3f9..5144109 100644 --- a/test/prefix.bats +++ b/test/prefix.bats @@ -12,10 +12,6 @@ create_version() { chmod +x "${PYENV_ROOT}/versions/$1/bin/python" } -remove_version() { - rm -fr "${PYENV_ROOT}/versions/$1" -} - create_virtualenv() { create_version "$1" create_version "${2:-$1}" @@ -40,11 +36,6 @@ create_virtualenv_pypy() { touch "${PYENV_ROOT}/versions/$1/bin/activate" } -remove_virtualenv() { - remove_version "$1" - remove_version "${2:-$1}" -} - create_m_venv() { create_version "$1" create_version "${2:-$1}" @@ -52,10 +43,6 @@ create_m_venv() { touch "${PYENV_ROOT}/versions/$1/bin/activate" } -remove_m_venv() { - remove_version "${2:-$1}" -} - create_conda() { create_version "$1" create_version "${2:-$1}" @@ -66,13 +53,8 @@ create_conda() { touch "${PYENV_ROOT}/versions/${2:-$1}/bin/activate" } -remove_conda() { - remove_version "${2:-$1}" -} - @test "display prefix of virtualenv created by virtualenv" { stub pyenv-version-name "echo foo" - stub pyenv-prefix "foo : echo \"${PYENV_ROOT}/versions/foo\"" create_virtualenv "foo" "2.7.11" PYENV_VERSION="foo" run pyenv-virtualenv-prefix @@ -83,13 +65,10 @@ ${PYENV_ROOT}/versions/2.7.11 OUT unstub pyenv-version-name - unstub pyenv-prefix - remove_virtualenv "foo" "2.7.11" } @test "display prefix of virtualenv created by virtualenv (pypy)" { stub pyenv-version-name "echo foo" - stub pyenv-prefix "foo : echo \"${PYENV_ROOT}/versions/foo\"" create_virtualenv_pypy "foo" "pypy-4.0.1" PYENV_VERSION="foo" run pyenv-virtualenv-prefix @@ -100,13 +79,10 @@ ${PYENV_ROOT}/versions/pypy-4.0.1 OUT unstub pyenv-version-name - unstub pyenv-prefix - remove_virtualenv "foo" "pypy-4.0.1" } @test "display prefix of virtualenv created by virtualenv (jython)" { stub pyenv-version-name "echo foo" - stub pyenv-prefix "foo : echo \"${PYENV_ROOT}/versions/foo\"" create_virtualenv_jython "foo" "jython-2.7.0" PYENV_VERSION="foo" run pyenv-virtualenv-prefix @@ -117,14 +93,10 @@ ${PYENV_ROOT}/versions/jython-2.7.0 OUT unstub pyenv-version-name - unstub pyenv-prefix - remove_virtualenv "foo" "jython-2.7.0" } @test "display prefixes of virtualenv created by virtualenv" { stub pyenv-version-name "echo foo:bar" - stub pyenv-prefix "foo : echo \"${PYENV_ROOT}/versions/foo\"" \ - "bar : echo \"${PYENV_ROOT}/versions/bar\"" create_virtualenv "foo" "2.7.11" create_virtualenv "bar" "3.5.1" @@ -136,14 +108,10 @@ ${PYENV_ROOT}/versions/2.7.11:${PYENV_ROOT}/versions/3.5.1 OUT unstub pyenv-version-name - unstub pyenv-prefix - remove_virtualenv "foo" "2.7.11" - remove_virtualenv "bar" "3.5.1" } @test "display prefix of virtualenv created by venv" { stub pyenv-version-name "echo foo" - stub pyenv-prefix "foo : echo \"${PYENV_ROOT}/versions/foo\"" create_m_venv "foo" "3.3.6" PYENV_VERSION="foo" run pyenv-virtualenv-prefix @@ -154,14 +122,10 @@ ${PYENV_ROOT}/versions/3.3.6 OUT unstub pyenv-version-name - unstub pyenv-prefix - remove_m_venv "foo" "3.3.6" } @test "display prefixes of virtualenv created by venv" { stub pyenv-version-name "echo foo:bar" - stub pyenv-prefix "foo : echo \"${PYENV_ROOT}/versions/foo\"" \ - "bar : echo \"${PYENV_ROOT}/versions/bar\"" create_m_venv "foo" "3.3.6" create_m_venv "bar" "3.4.4" @@ -173,14 +137,10 @@ ${PYENV_ROOT}/versions/3.3.6:${PYENV_ROOT}/versions/3.4.4 OUT unstub pyenv-version-name - unstub pyenv-prefix - remove_m_venv "foo" "3.3.6" - remove_m_venv "bar" "3.4.4" } @test "display prefix of virtualenv created by conda" { stub pyenv-version-name "echo miniconda3-3.16.0/envs/foo" - stub pyenv-prefix "miniconda3-3.16.0/envs/foo : echo \"${PYENV_ROOT}/versions/miniconda3-3.16.0/envs/foo\"" create_conda "miniconda3-3.16.0/envs/foo" "miniconda3-3.16.0" PYENV_VERSION="miniconda3-3.16.0/envs/foo" run pyenv-virtualenv-prefix @@ -191,8 +151,6 @@ ${PYENV_ROOT}/versions/miniconda3-3.16.0/envs/foo OUT unstub pyenv-version-name - unstub pyenv-prefix - remove_conda "miniconda3-3.16.0/envs/foo" "miniconda3-3.16.0" } @test "should fail if the version is the system" { @@ -210,7 +168,6 @@ OUT @test "should fail if the version is not a virtualenv" { stub pyenv-version-name "echo 3.4.4" - stub pyenv-prefix "3.4.4 : echo \"${PYENV_ROOT}/versions/3.4.4\"" create_version "3.4.4" PYENV_VERSION="3.4.4" run pyenv-virtualenv-prefix @@ -221,14 +178,10 @@ pyenv-virtualenv: version \`3.4.4' is not a virtualenv OUT unstub pyenv-version-name - unstub pyenv-prefix - remove_version "3.4.4" } @test "should fail if one of the versions is not a virtualenv" { stub pyenv-version-name "echo venv33:3.4.4" - stub pyenv-prefix "venv33 : echo \"${PYENV_ROOT}/versions/venv33\"" \ - "3.4.4 : echo \"${PYENV_ROOT}/versions/3.4.4\"" create_virtualenv "venv33" "3.3.6" create_version "3.4.4" @@ -240,7 +193,22 @@ pyenv-virtualenv: version \`3.4.4' is not a virtualenv OUT unstub pyenv-version-name - unstub pyenv-prefix - remove_virtualenv "venv33" "3.3.6" - remove_version "3.4.4" } + +@test "resolves a version that is not an exact match" { + stub pyenv-version-name "echo 3" + create_version "3.4.4" + + stub pyenv-prefix "3 : echo \"$PYENV_ROOT/versions/3.4.4\"" + + run pyenv-virtualenv-prefix + + assert_failure + assert_output < "${venv_dir}/pyvenv.cfg" } -@test "list virtual environments only" { - stub pyenv-version-name ": echo system" - stub pyenv-virtualenv-prefix "2.7.6 : false" - stub pyenv-virtualenv-prefix "3.3.3 : false" - stub pyenv-virtualenv-prefix "venv27 : echo \"${PYENV_ROOT}/versions/2.7.6\"" - stub pyenv-virtualenv-prefix "venv33 : echo \"${PYENV_ROOT}/versions/3.3.3\"" + +create_m_venv() { + # Create a mock virtual environment from an installed (not system) Python version. + # + # The venv name is a symlink inside ${PYENV_ROOT}/versions that points to + # the real venv directory inside the Python version used to created and + # home points to the bin directory inside the venv dir. + setup_m_venv "$1/envs/$2" + + local version="$1" + local venv="$2" + local venv_dir="${PYENV_ROOT}/versions/${version}/envs/${venv}" + + echo "home = ${PYENV_ROOT}/versions/${version}/bin" > "${venv_dir}/pyvenv.cfg" + ln -sf "${venv_dir}" "${PYENV_ROOT}/versions/${venv}" +} + +setup() { + export PYENV_ROOT="${TMP}/pyenv" +} + +@test "system venv" { + stub pyenv-version-name ": echo venv314" + stub pyenv-version-origin ": echo PYENV_VERSION" + + create_m_venv "3.14.3" "venv314" + create_m_system_venv "3.9.11" "system_venv" run pyenv-virtualenvs assert_success assert_output < ${PYENV_ROOT}/versions/3.14.3/envs/venv314 (set by PYENV_VERSION) +OUT + + unstub pyenv-version-name + unstub pyenv-version-origin +} + +@test "list virtual environments" { + stub pyenv-version-name ": echo system" + stub pyenv-virtualenv-prefix "2.7.6/envs/venv27 : echo \"${PYENV_ROOT}/versions/2.7.6\"" + stub pyenv-virtualenv-prefix "3.3.3/envs/venv33 : echo \"${PYENV_ROOT}/versions/3.3.3\"" + + create_m_venv "2.7.6" "venv27" + create_m_venv "3.3.3" "venv33" + + run pyenv-virtualenvs + + assert_success + assert_output < ${PYENV_ROOT}/versions/2.7.6/envs/venv27 + venv33 --> ${PYENV_ROOT}/versions/3.3.3/envs/venv33 OUT unstub pyenv-version-name @@ -30,37 +82,101 @@ OUT } @test "list virtual environments with hit prefix" { - stub pyenv-version-name ": echo venv33" - stub pyenv-virtualenv-prefix "2.7.6 : false" - stub pyenv-virtualenv-prefix "3.3.3 : false" - stub pyenv-virtualenv-prefix "venv27 : echo \"/usr\"" - stub pyenv-virtualenv-prefix "venv33 : echo \"/usr\"" + stub pyenv-version-name ": echo 3.3.3/envs/venv33" + stub pyenv-version-origin ": echo PYENV_VERSION" + + create_m_venv "2.7.6" "venv27" + create_m_venv "3.3.3" "venv33" run pyenv-virtualenvs assert_success assert_output < ${PYENV_ROOT}/versions/2.7.6/envs/venv27 + venv33 --> ${PYENV_ROOT}/versions/3.3.3/envs/venv33 OUT unstub pyenv-version-name - unstub pyenv-virtualenv-prefix + unstub pyenv-version-origin } -@test "list virtual environments with --bare" { - stub pyenv-virtualenv-prefix "2.7.6 : false" - stub pyenv-virtualenv-prefix "3.3.3 : false" - stub pyenv-virtualenv-prefix "venv27 : echo \"/usr\"" - stub pyenv-virtualenv-prefix "venv33 : echo \"/usr\"" +@test "list bare virtual environments" { + create_m_venv "2.7.6" "venv27" + create_m_venv "3.3.3" "venv33" run pyenv-virtualenvs --bare assert_success assert_output < ${PYENV_ROOT}/versions/2.7.6/envs/venv27 (set by PYENV_VERSION) + venv33 --> ${PYENV_ROOT}/versions/3.3.3/envs/venv33 +OUT + + unstub pyenv-version-name + unstub pyenv-version-origin +} + +@test "completions output" { + create_m_venv "2.7.6" "venv27" + create_m_venv "3.3.3" "venv33" + + run pyenv-virtualenvs --complete + + assert_success + assert_output <