From c30fbf900282603e6e1ad0e54e05ff15daaa196c Mon Sep 17 00:00:00 2001 From: Nick Bolton Date: Fri, 6 Sep 2024 13:50:59 +0100 Subject: [PATCH] Use `.venv` dir for as Python venv and cache (#7485) * Use .venv as Python venv dir * Update refs to Python venv dir and use action to cache and setup * Add missing shell: bash * Source for Python deps * Exclude .venv from lint * Update ChangeLog * Add cache-key arg for init-python * Add missing " * Use workflow specific Python cache names * Fixed cache key for Linux * Use bash if to make output clearer in case of skipping. * Clearer debug output * Add check for cache key * Add missing shell * Add SonarCloud and Valgrind venv cache * Fixed typo --- .clang-format | 4 +- .github/actions/init-python/action.yml | 50 +++++++++++++++++++ .github/workflows/ci.yml | 23 +++++++-- .github/workflows/codeql-analysis.yml | 7 +++ ...stale-issues.yml => issue-check-stale.yml} | 2 +- ...ue-support.yml => issue-check-support.yml} | 9 +++- .github/workflows/lint-source-code.yml | 8 ++- .github/workflows/sonarcloud-analysis.yml | 7 +++ .github/workflows/valgrind-analysis.yml | 7 +++ .gitignore | 3 +- ChangeLog | 1 + cmake/Libraries.cmake | 5 +- cspell.json | 3 ++ scripts/lib/env.py | 15 +++--- scripts/lint_cmake.py | 11 ++-- 15 files changed, 131 insertions(+), 24 deletions(-) create mode 100644 .github/actions/init-python/action.yml rename .github/workflows/{stale-issues.yml => issue-check-stale.yml} (96%) rename .github/workflows/{issue-support.yml => issue-check-support.yml} (89%) diff --git a/.clang-format b/.clang-format index 06204b36e..6a994d72f 100644 --- a/.clang-format +++ b/.clang-format @@ -1,7 +1,7 @@ # Important: Use the same version of clang-format as our linter, which is: # # $ ./scripts/install_deps.py --only-python -# $ ./build/python/bin/clang-format --version +# $ ./.venv/bin/clang-format --version # # Warning: If you use a different version, the formatting will be different. # @@ -10,7 +10,7 @@ BasedOnStyle: LLVM -# Turn off LLVM default alignment of params with the opening bracket, +# Turn off LLVM default alignment of params with the opening bracket, # which can be less readable in some cases in our code base. # # Using `AlwaysBreak` will result in: diff --git a/.github/actions/init-python/action.yml b/.github/actions/init-python/action.yml new file mode 100644 index 000000000..cd14da4f5 --- /dev/null +++ b/.github/actions/init-python/action.yml @@ -0,0 +1,50 @@ +name: "Setup Python venv" +description: "Creates and caches a Python virtual environment" + +inputs: + cache: + description: "Cache Python venv" + default: true + + setup: + description: "Setup Python venv" + default: true + + python-bin: + description: "Python binary to use" + default: "python3" + + cache-key: + description: "Cache key (note: hash is appended)" + required: true + +runs: + using: "composite" + + steps: + - name: Check cache key + if: ${{ inputs.cache }} + run: | + if [ -z "${{ inputs.cache-key }}" ]; then + echo "Cache key is required" + exit 1 + fi + shell: bash + + - name: Cache Python venv + if: ${{ inputs.cache }} + uses: actions/cache@v4 + with: + path: .venv + key: python_venv-${{ inputs.cache-key }}-${{ hashFiles('scripts/pyproject.toml') }} + + # Use bash if to make output clearer in case of skipping. + - name: Setup Python venv + run: | + if [ "${{ inputs.setup }}" = "true" ]; then + echo "Setting up Python venv" + ${{ inputs.python-bin }} -m venv .venv + else + echo "Skipping Python venv setup" + fi + shell: bash diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ecf16de78..4031f3875 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -71,11 +71,12 @@ jobs: key: vcpkg-${{ runner.os }}-${{ hashFiles('vcpkg.json', 'vcpkg-configuration.json') }} restore-keys: vcpkg-${{ runner.os }} - - name: Cache Python env - uses: actions/cache@v4 + # Should only restore the .venv directory from cache. + - name: Init Python venv + uses: ./.github/actions/init-python with: - path: build/python - key: python_env-${{ runner.os }}-${{ hashFiles('scripts/pyproject.toml') }} + cache-key: ci-${{ matrix.target.name }} + setup: false - name: Cache deps dir uses: actions/cache@v4 @@ -163,6 +164,13 @@ jobs: - name: Get version uses: ./.github/actions/get-version + # Should only restore the .venv directory from cache. + - name: Setup Python venv + uses: ./.github/actions/init-python + with: + cache-key: ci-${{ matrix.target.name }} + setup: false + - name: Cache deps dir uses: actions/cache@v4 with: @@ -281,6 +289,13 @@ jobs: - name: Checkout uses: actions/checkout@v4 + # Should only restore the .venv directory from cache. + - name: Setup Python venv + uses: ./.github/actions/init-python + with: + cache-key: ci-${{ matrix.distro.name }} + setup: false + - name: Get version uses: ./.github/actions/get-version diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 0bfa57732..1de8deb50 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -32,6 +32,13 @@ jobs: - name: Config Git safe dir run: git config --global --add safe.directory $GITHUB_WORKSPACE + # Should only restore the .venv directory from cache. + - name: Init Python venv + uses: ./.github/actions/init-python + with: + cache-key: "codeql" + setup: false + - name: Install dependencies run: ./scripts/install_deps.py env: diff --git a/.github/workflows/stale-issues.yml b/.github/workflows/issue-check-stale.yml similarity index 96% rename from .github/workflows/stale-issues.yml rename to .github/workflows/issue-check-stale.yml index b3467df1f..c16316011 100644 --- a/.github/workflows/stale-issues.yml +++ b/.github/workflows/issue-check-stale.yml @@ -1,4 +1,4 @@ -name: "Close stale issues and PRs" +name: Close stale issues and PRs on: schedule: - cron: "0 5 * * *" diff --git a/.github/workflows/issue-support.yml b/.github/workflows/issue-check-support.yml similarity index 89% rename from .github/workflows/issue-support.yml rename to .github/workflows/issue-check-support.yml index 011ceee2c..a3e420fa6 100644 --- a/.github/workflows/issue-support.yml +++ b/.github/workflows/issue-check-support.yml @@ -12,14 +12,19 @@ on: required: true jobs: - check-issue-tech-support: + issue-check-support: runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 - - name: Check if issue requires tech support + - name: Setup Python venv + uses: ./.github/actions/ + with: + cache-key: "ubuntu-latest" + + - name: Check issue id: issue-check run: ./scripts/github.py --issue-check-tech-support env: diff --git a/.github/workflows/lint-source-code.yml b/.github/workflows/lint-source-code.yml index 535a8181e..be1ea5143 100644 --- a/.github/workflows/lint-source-code.yml +++ b/.github/workflows/lint-source-code.yml @@ -20,10 +20,14 @@ jobs: - name: Checkout uses: actions/checkout@v4 + - name: Setup Python venv + uses: ./.github/actions/init-python + with: + cache-key: "lint-source-code" + - name: Install dependencies run: | - python3 -m venv build/python - source build/python/bin/activate + source .venv/bin/activate pip install pyyaml cmake_format clang_format - name: Linting with CMake formatter diff --git a/.github/workflows/sonarcloud-analysis.yml b/.github/workflows/sonarcloud-analysis.yml index 0faee0d96..8e7149df7 100644 --- a/.github/workflows/sonarcloud-analysis.yml +++ b/.github/workflows/sonarcloud-analysis.yml @@ -35,6 +35,13 @@ jobs: - name: Config Git safe dir run: git config --global --add safe.directory $GITHUB_WORKSPACE + # Should only restore the .venv directory from cache. + - name: Init Python venv + uses: ./.github/actions/init-python + with: + cache-key: "sonarcloud" + setup: false + - name: Install dependencies run: | ./scripts/install_deps.py && diff --git a/.github/workflows/valgrind-analysis.yml b/.github/workflows/valgrind-analysis.yml index ba6828f09..8de54c4be 100644 --- a/.github/workflows/valgrind-analysis.yml +++ b/.github/workflows/valgrind-analysis.yml @@ -22,6 +22,13 @@ jobs: - name: Config Git safe dir run: git config --global --add safe.directory $GITHUB_WORKSPACE + # Should only restore the .venv directory from cache. + - name: Init Python venv + uses: ./.github/actions/init-python + with: + cache-key: "valgrind" + setup: false + - name: Install dependencies run: | ./scripts/install_deps.py && diff --git a/.gitignore b/.gitignore index f588f0274..2f6577df0 100644 --- a/.gitignore +++ b/.gitignore @@ -6,9 +6,10 @@ /vcpkg /vcpkg_installed /scripts/**/*.pyc +/.cache +/.venv aqtinstall.log Brewfile.lock.json -/.cache # typical developer-created files synergy-config.toml diff --git a/ChangeLog b/ChangeLog index 875c08edb..ebb334c03 100644 --- a/ChangeLog +++ b/ChangeLog @@ -16,6 +16,7 @@ Enhancements: - #7473 Simplify `vcpkg` caching and use system `vcpkg` - #7474 FreeBSD GitHub runner with `vmactions/freebsd-vm@v1` - #7479 Add `BUILD.md` to get people started +- #7485 Use `.venv` dir for as Python venv and cache # 1.15.1 diff --git a/cmake/Libraries.cmake b/cmake/Libraries.cmake index c1c8206a4..cc133b7c3 100644 --- a/cmake/Libraries.cmake +++ b/cmake/Libraries.cmake @@ -600,10 +600,11 @@ macro(configure_coverage) endmacro() macro(configure_python) + set(python_venv_dir ${CMAKE_SOURCE_DIR}/.venv) if(WIN32) - set(PYTHON_BIN "${CMAKE_BINARY_DIR}/python/Scripts/python.exe") + set(PYTHON_BIN ${python_venv_dir}/Scripts/python.exe) else() - set(PYTHON_BIN "${CMAKE_BINARY_DIR}/python/bin/python") + set(PYTHON_BIN ${python_venv_dir}/bin/python) endif() endmacro() diff --git a/cspell.json b/cspell.json index 06c11551b..68bd9e767 100644 --- a/cspell.json +++ b/cspell.json @@ -7,6 +7,7 @@ "aqtinstall", "Axelson", "Breen", + "cmakelang", "codesign", "codesigning", "Compat", @@ -64,6 +65,8 @@ "unittests", "Valgrind", "vcpkg", + "venv", + "vmactions", "Volker", "whot", "winget" diff --git a/scripts/lib/env.py b/scripts/lib/env.py index 2bd53b9db..4a208816b 100644 --- a/scripts/lib/env.py +++ b/scripts/lib/env.py @@ -16,7 +16,8 @@ import os, sys, subprocess import lib.cmd_utils as cmd_utils -venv_path = "build/python" +# The `.venv` dir seems to be most common for virtual environments. +VENV_DIR = ".venv" def check_module(module): @@ -105,9 +106,9 @@ def get_env_bool(name, default=False): def get_python_executable(binary="python"): if sys.platform == "win32": - return os.path.join(venv_path, "Scripts", binary) + return os.path.join(VENV_DIR, "Scripts", binary) else: - return os.path.join(venv_path, "bin", binary) + return os.path.join(VENV_DIR, "bin", binary) def in_venv(): @@ -125,13 +126,13 @@ def ensure_in_venv(script_file, auto_create=False): import venv if not in_venv(): - if not os.path.exists(venv_path): + if not os.path.exists(VENV_DIR): if not auto_create: print("Hint: Run the `install_deps.py` script first.") - raise RuntimeError(f"Virtual environment not found at: {venv_path}") + raise RuntimeError(f"Virtual environment not found at: {VENV_DIR}") - print(f"Creating virtual environment at {venv_path}") - venv.create(venv_path, with_pip=True) + print(f"Creating virtual environment at {VENV_DIR}") + venv.create(VENV_DIR, with_pip=True) script_file_abs = os.path.abspath(script_file) print(f"Using virtual environment for: {script_file_abs}", flush=True) diff --git a/scripts/lint_cmake.py b/scripts/lint_cmake.py index 6e292b72a..86aedbc40 100755 --- a/scripts/lint_cmake.py +++ b/scripts/lint_cmake.py @@ -23,12 +23,17 @@ import sys, argparse import lib.fs as fs from cmakelang.format.__main__ import main as cmake_format_main # type: ignore -include_files = [ +INCLUDE_FILES = [ "*.cmake", "CMakeLists.txt", ] -exclude_dirs = ["subprojects", "build", "deps"] +EXCLUDE_DIRS = [ + "build", + ".venv", + "deps", + "subprojects", +] def main(): @@ -46,7 +51,7 @@ def main(): args = parser.parse_args() cmd_args = ["--in-place"] if args.format else ["--check"] - files_recursive = fs.find_files(".", include_files, exclude_dirs) + files_recursive = fs.find_files(".", INCLUDE_FILES, EXCLUDE_DIRS) if args.format: print("Formatting files with CMake formatter:")