From d0a4c3d7a05d99c31a9bc05e3ebb82dcbb77f1e7 Mon Sep 17 00:00:00 2001 From: Nick Bolton Date: Fri, 14 Jun 2024 16:32:47 +0100 Subject: [PATCH] Use deps script to make life easier for contribs (#7351) * Use deps script to make life easier for contribs * Make readme more developer friendly * YAML file to describe Linux deps * Silent fail for yaml module (Python 3.6+) * Move Qt install to deps script * Update readme to add Windows command * Fixed Linux package name * Simplify env vars for workflow * Exclude scripts from SonarCloud * Move all deps config to .yml file * Fixed exclude glob * Simplify quickstart * Fixed macOS deps config key name * Fixed bug in Linux config read * Use backslash for consistency * Change SonarCloud path to /build/ * Fixed dep name * Exclude scripts in CMake coverage * Move script coverage exclusion to online config * Use try/except instead of checking return value * Fixed comment typo --- .github/workflows/job-test-mac.yml | 8 +- .github/workflows/job-test-windows.yml | 14 +- .github/workflows/sonarcloud-analysis.yml | 14 +- .gitignore | 2 + Brewfile | 4 + ChangeLog | 1 + Chocolatey.config | 11 + README.md | 61 +++- deps.yml | 87 +++++ scripts/install_deps.py | 367 ++++++++++++++++++---- 10 files changed, 476 insertions(+), 93 deletions(-) create mode 100644 Brewfile create mode 100644 Chocolatey.config create mode 100644 deps.yml diff --git a/.github/workflows/job-test-mac.yml b/.github/workflows/job-test-mac.yml index 857e7047b..b4adfc424 100644 --- a/.github/workflows/job-test-mac.yml +++ b/.github/workflows/job-test-mac.yml @@ -13,7 +13,7 @@ concurrency: jobs: test-macos: runs-on: ${{ matrix.runtime.os }} - timeout-minutes: 10 + timeout-minutes: 20 defaults: run: @@ -54,8 +54,10 @@ jobs: with: submodules: "recursive" - - name: Install Qt5 - run: brew install qt@5 + - name: Install dependencies + run: | + pip install pyyaml + python ./scripts/install_deps.py - name: Configure env: diff --git a/.github/workflows/job-test-windows.yml b/.github/workflows/job-test-windows.yml index cea68f2d4..36529e94b 100644 --- a/.github/workflows/job-test-windows.yml +++ b/.github/workflows/job-test-windows.yml @@ -20,8 +20,6 @@ jobs: BONJOUR_BASE_DIR: ${{ github.workspace }}\deps\bonjour QT_BASE_DIR: ${{ github.workspace }}\deps\Qt QT_VERSION: 5.15.2 - QT_BASE_URL: http://qt.mirror.constant.com/ - QT_LIB_DIR: ${{ github.workspace }}\deps\Qt\5.15.2 steps: - name: Checkout git repo @@ -38,11 +36,7 @@ jobs: - name: Install Qt if: steps.cache-qt.outputs.cache-hit != 'true' - run: | - pip install aqtinstall - python -m aqt install --outputdir $env:QT_BASE_DIR --base $env:QT_BASE_URL $env:QT_VERSION windows desktop win64_msvc2019_64 - cd $env:QT_LIB_DIR\msvc2019_64 - dir + run: python ./scripts/install_deps.py --only qt - name: Cache Bonjour id: cache-bonjour @@ -60,14 +54,16 @@ jobs: [System.IO.Compression.ZipFile]::ExtractToDirectory(".\bonjoursdk.zip", "$env:BONJOUR_BASE_DIR") - name: Install dependencies - run: python ./scripts/install_deps.py --skip cmake ninja + run: | + pip install pyyaml + python ./scripts/install_deps.py - name: Add msbuild to PATH uses: microsoft/setup-msbuild@v1.0.2 - name: Build env: - CMAKE_PREFIX_PATH: "${{ env.QT_LIB_DIR }}\\msvc2019_64\\" + CMAKE_PREFIX_PATH: "${{ env.QT_BASE_DIR }}\\${{ env.QT_VERSION }}\\msvc2019_64\\" run: | mkdir build cd build diff --git a/.github/workflows/sonarcloud-analysis.yml b/.github/workflows/sonarcloud-analysis.yml index 6d2fbaf8a..11e65cf99 100644 --- a/.github/workflows/sonarcloud-analysis.yml +++ b/.github/workflows/sonarcloud-analysis.yml @@ -38,7 +38,7 @@ jobs: curl --create-dirs -sSLo $HOME/.sonar/sonar-scanner.zip https://binaries.sonarsource.com/Distribution/sonar-scanner-cli/sonar-scanner-cli-$SONAR_SCANNER_VERSION-linux.zip unzip -o $HOME/.sonar/sonar-scanner.zip -d $HOME/.sonar/ - - name: Installing buildWrapper + - name: Installing build-wrapper run: | curl --create-dirs -sSLo $HOME/.sonar/build-wrapper-linux-x86.zip https://sonarcloud.io/static/cpp/build-wrapper-linux-x86.zip unzip -o $HOME/.sonar/build-wrapper-linux-x86.zip -d $HOME/.sonar/ @@ -46,30 +46,30 @@ jobs: - name: Build run: | export PATH=$HOME/.sonar/build-wrapper-linux-x86:$PATH - mkdir build-release - cd build-release + mkdir build + cd build cmake -DCMAKE_BUILD_TYPE=Debug -DENABLE_COVERAGE=ON .. . ./version build-wrapper-linux-x86-64 --out-dir bw-output make -j - name: Running coverage run: | - cd build-release + cd build make coverage - name: Run Sonar Scanner run: | export PATH=$HOME/.sonar/sonar-scanner-${SONAR_SCANNER_VERSION}-linux/bin:$PATH - cd build-release + cd build sonar-scanner \ -Dsonar.organization=symless \ -Dsonar.projectKey=symless_synergy-core \ -Dsonar.sources=. \ -Dsonar.projectBaseDir=../ \ - -Dsonar.exclusions=./ext/**,**/build-release/** \ + -Dsonar.exclusions=ext/**,build/** \ -Dsonar.cfamily.build-wrapper-output=bw-output \ -Dsonar.host.url=https://sonarcloud.io \ - -Dsonar.coverageReportPaths=build-release/coverage.xml \ + -Dsonar.coverageReportPaths=build/coverage.xml \ -Dsonar.cfamily.threads=2 env: SONAR_TOKEN: ${{secrets.SONAR_TOKEN}} diff --git a/.gitignore b/.gitignore index 612af41f5..862e2cb00 100644 --- a/.gitignore +++ b/.gitignore @@ -34,3 +34,5 @@ flatpak/synergy.desktop flatpak/*.flatpak *.code-workspace *.idx +aqtinstall.log +Brewfile.lock.json diff --git a/Brewfile b/Brewfile new file mode 100644 index 000000000..a63867112 --- /dev/null +++ b/Brewfile @@ -0,0 +1,4 @@ +brew 'make' +brew 'cmake' +brew 'openssl' +brew 'qt@5' diff --git a/ChangeLog b/ChangeLog index f23e3a846..ed43cd8d9 100644 --- a/ChangeLog +++ b/ChangeLog @@ -31,6 +31,7 @@ Enhancements: - #7334 Implement hello back in IPC protocol - #7335 Clickable debug source paths and new launch target - #7336 Add C++ and LLDB to VS Code recommendations +- #7351 Use deps script to make life easier for contribs # 1.14.6 diff --git a/Chocolatey.config b/Chocolatey.config new file mode 100644 index 000000000..b03282198 --- /dev/null +++ b/Chocolatey.config @@ -0,0 +1,11 @@ + + + + + + + + + + + \ No newline at end of file diff --git a/README.md b/README.md index c7a4bdc93..f734788a2 100644 --- a/README.md +++ b/README.md @@ -2,25 +2,56 @@ This is the open source core component of Synergy, a keyboard and mouse sharing tool. -## Recommended +* [Download Synergy](https://symless.com/synergy/download) +* [Contact support](https://symless.com/synergy/contact) +* [Help articles](https://symless.com/synergy/help) -Things most people will need. +## Developer quick start -* [Download](https://symless.com/synergy/download) - Get the compiled version of Synergy 3 and Synergy 1. -* [Contact Support](https://symless.com/contact-support) - Open a support ticket and talk directly to the Synergy team. -* [Help Guides](https://symless.com/help) - Self-help guides and information for when you don't want to talk to people. -* [Symless Blog](https://symless.com/blog/) - Find out what's happening at Symless and with Synergy development. +Simplified instructions for those who want to contribute to development of Synergy Core. -## Advanced Users +You'll also need to read the +[quick start companion](https://github.com/symless/synergy-core/wiki/Quick-start-companion) +which contains essential instructions. + +**Dependencies:** +``` +python scripts/install_deps.py +``` + +**Configure:** + +*Windows:* +``` +cmake -B build --preset=windows-release +``` + +*macOS:* +``` +cmake -B build --preset=macos-release +``` + +*Linux:* +``` +cmake -B build --preset=linux-release +``` + +**Build:** +``` +cmake --build build +``` + +**Test:** +``` +./build/bin/unittests +``` + +## Developer resources For people who want to contribute to the development of Synergy. -* [Getting Started](https://github.com/symless/synergy-core/wiki/Getting-Started) - How to checkout the code from git and use the right branch. +* [Getting started](https://github.com/symless/synergy-core/wiki/Getting-Started) - How to checkout the code from git and use the right branch. * [Compiling](https://github.com/symless/synergy-core/wiki/Compiling) - Instructions on how to compile Synergy Core from source. -* [Text Config](https://github.com/symless/synergy-core/wiki/Text-Config) - Write a text config file when running Synergy Core manually. -* [Command Line](https://github.com/symless/synergy-core/wiki/Command-Line) - Go full manual and run Synergy Core from the command line. - -## Synergy Vintage - -For vintage computer enthusiasts, [Synergy Vintage](https://github.com/nbolton/synergy-vintage) aims to keep the origins of Synergy alive. -You can use Synergy Vintage on operating systems available from 1995 to 2006. +* [Text config](https://github.com/symless/synergy-core/wiki/Text-Config) - Write a text config file when running Synergy Core manually. +* [Command line](https://github.com/symless/synergy-core/wiki/Command-Line) - Go full manual and run Synergy Core from the command line. +* [Synergy Vintage](https://github.com/nbolton/synergy-vintage) - Use Synergy on operating systems available between 1995 and 2006. diff --git a/deps.yml b/deps.yml new file mode 100644 index 000000000..9da4a8d43 --- /dev/null +++ b/deps.yml @@ -0,0 +1,87 @@ +dependencies: + windows: + command: choco install Chocolatey.config -y + qt: + version: 5.15.2 + mirror: https://qt.mirror.constant.com/ + install-dir: C:\Qt + + mac: + command: brew bundle --file=Brewfile + + linux: + debian: &debian + command: sudo apt-get update && sudo apt-get install -y + packages: + - cmake + - make + - xorg-dev + - libx11-dev + - libxtst-dev + - libssl-dev + - pkg-config + - libglib2.0-dev + - libgdk-pixbuf-2.0-dev + - libnotify-dev + - libxkbfile-dev + - qtbase5-dev + - qttools5-dev + - libgtk-3-dev + - rpm + + ubuntu: + <<: *debian + + fedora: &fedora + command: sudo dnf install -y + packages: + - cmake + - make + - gcc-c++ + - libXtst-devel + - glib2-devel + - gdk-pixbuf2-devel + - libnotify-devel + - qt5-qtbase-devel + - libxkbfile-devel + - gtk3-devel + - rpm-build + + centos: + <<: *fedora + command: sudo yum install -y + + # Warning: AI generated (untested) + arch: + command: sudo pacman -Syu --noconfirm + packages: + - cmake + - make + - gcc + - libx11 + - libxtst + - openssl + - pkg-config + - glib2 + - gdk-pixbuf2 + - libnotify + - libxkbfile + - qt5-base + - gtk3 + - rpm + + # Warning: AI generated (untested) + opensuse: + command: sudo zypper install -y + packages: + - cmake + - make + - gcc-c++ + - libXtst-devel + - glib2-devel + - gdk-pixbuf-devel + - libnotify-devel + - libxkbfile-devel + - libqt5-qtbase-devel + - gtk3-devel + - rpm-build diff --git a/scripts/install_deps.py b/scripts/install_deps.py index 2de01f1ba..9f498c624 100644 --- a/scripts/install_deps.py +++ b/scripts/install_deps.py @@ -3,85 +3,334 @@ from lib import windows import subprocess import sys import argparse +import traceback + +config_file = "deps.yml" + + +class EnvError(Exception): + pass + + +class YamlError(Exception): + pass + + +class PlatformError(Exception): + pass + + +class PathError(Exception): + pass + + +try: + import yaml # type: ignore +except ImportError: + # this is fairly common in earlier versions of python3, + # which is normally what you find on mac and windows. + print("Python yaml module missing, please install: pip install pyyaml") + sys.exit(1) + def main(): - """Entry point for the script.""" + """Entry point for the script.""" - parser = argparse.ArgumentParser() - parser.add_argument('--pause-on-exit', action='store_true') - parser.add_argument('--skip', nargs='*', default=[]) - args = parser.parse_args() + parser = argparse.ArgumentParser() + parser.add_argument( + "--pause-on-exit", action="store_true", help="Useful on Windows" + ) + parser.add_argument( + "--only", type=str, help="Only install the specified dependency" + ) + args = parser.parse_args() - try: - deps = Deps(args.skip) - deps.install() - except Exception as e: - print(f'Error: {e}') - - if (args.pause_on_exit): - input('Press enter to continue...') + try: + deps = Dependencies(args.only) + deps.install() + except Exception: + traceback.print_exc() -class Deps: + if args.pause_on_exit: + input("Press enter to continue...") - def __init__(self, skip): - self.skip = skip - def install(self): - """Installs dependencies.""" +def run(command, check=True): + """Runs a shell command and by default asserts that the return code is 0.""" - if (sys.platform == 'win32'): - self.windows() + command_str = command + if isinstance(command, list): + command_str = " ".join(command) + + print(f"Running: {command_str}") + + try: + subprocess.run(command, shell=True, check=check) + except subprocess.CalledProcessError as e: + print(f"Command failed: {command_str}", file=sys.stderr) + raise e + + +def get_os(): + """Detects the operating system.""" + if sys.platform == "win32": + return "windows" + elif sys.platform == "darwin": + return "mac" + elif sys.platform.startswith("linux"): + return "linux" else: - print(f'Unsupported platform: {sys.platform}') + raise PlatformError(f"Unsupported platform: {sys.platform}") - def windows(self): - """Installs dependencies on Windows.""" - if not windows.is_admin(): - windows.relaunch_as_admin(__file__) - sys.exit() +def get_linux_distro(): + """Detects the Linux distro.""" + os_file = "/etc/os-release" + if os.path.isfile(os_file): + with open(os_file) as f: + for line in f: + if line.startswith("ID="): + return line.strip().split("=")[1].strip('"') + return None - ci_env = os.environ.get('CI') - if ci_env: - print('CI environment detected') - self.choco_ci() - # already installed on github runners. - self.skip.extend(['cmake', 'ninja']) - - self.choco("cmake") - self.choco("ninja") +class Config: + """Reads the dependencies configuration file.""" - # lock openssl to 3.1.1. as of 19th jan 2024, 3.2.0 breaks cmake configure. - self.choco("openssl", "3.1.1") + def __init__(self): + with open(config_file, "r") as f: + data = yaml.safe_load(f) - def choco(self, package, version=None): - """Installs a package using Chocolatey.""" + os_name = get_os() + try: + root = data["dependencies"] + except KeyError: + raise YamlError(f"Nothing found in {config_file} for: dependencies") - if (package in self.skip): - print(f'Skipping: {package}') - return - - args = ['choco', 'install', package] + try: + self.os = root[os_name] + except KeyError: + raise YamlError(f"Nothing found in {config_file} for: {os_name}") - if (version): - args.extend(['--version', version]) + def get_qt_config(self): + try: + return self.os["qt"] + except KeyError: + raise YamlError(f"Nothing found in {config_file} for: qt") - args.extend(['-y', '--no-progress']) + def get_packages_file(self): + try: + return self.os["packages"] + except KeyError: + raise YamlError(f"Nothing found in {config_file} for: packages") - subprocess.run(args, shell=True, check=True) + def get_linux_package_command(self, distro): + try: + distro_data = self.os[distro] + except KeyError: + raise YamlError(f"Nothing found in {config_file} for: {distro}") - def choco_ci(self): - """Configures Chocolatey cache for CI.""" + try: + command_base = distro_data["command"] + except KeyError: + raise YamlError(f"No package command found in {config_file} for: {distro}") + + try: + package_data = distro_data["packages"] + except KeyError: + raise YamlError(f"No package list found in {config_file} for: {distro}") + + packages = " ".join(package_data) + return f"{command_base} {packages}" + + +class Dependencies: + + def __init__(self, only): + self.config = Config() + self.only = only + + def install(self): + """Installs dependencies for the current platform.""" + + os = get_os() + if os == "windows": + self.windows() + elif os == "mac": + self.mac() + elif os == "linux": + self.linux() + else: + raise PlatformError(f"Unsupported platform: {os}") + + def windows(self): + """Installs dependencies on Windows.""" + + if not windows.is_admin(): + windows.relaunch_as_admin(__file__) + sys.exit() + + ci_env = os.environ.get("CI") + if ci_env: + print("CI environment detected") + + only_qt = self.only == "qt" + + # for ci, skip qt; we install qt separately so we can cache it. + if not ci_env or only_qt: + qt = WindowsQt(self.config.get_qt_config()) + qt_install_dir = qt.get_install_dir() + if qt_install_dir: + print(f"Skipping Qt, already installed at: {qt_install_dir}") + else: + qt.install() + + if only_qt: + return + + # use winget instead of choco to install the vc++ deps, since in choco there is no way + # to load a choco config file but skip a specific package (i.e. to skip vc++ for ci) + if not ci_env: + winget = WindowsWinGet() + winget.install_visual_studio() + + choco = WindowsChoco() + if ci_env: + choco.config_ci_cache() + + try: + command = self.config.os["command"] + except KeyError: + raise YamlError(f"Nothing found in {config_file} on Windows for: command") + + choco.install(command, ci_env) + + def mac(self): + """Installs dependencies on macOS.""" + try: + command = self.config.os["command"] + except KeyError: + raise YamlError(f"Nothing found in {config_file} on Mac for: command") + + run(command) + + def linux(self): + """Installs dependencies on Linux.""" + + distro = get_linux_distro() + if not distro: + raise PlatformError("Unable to detect Linux distro") + + command = self.config.get_linux_package_command(distro) + run(command) + + +class WindowsWinGet: + """WinGet for Windows.""" + + def install_visual_studio(self): + """Installs packages using WinGet.""" + + override = [ + "--quiet", + "--wait", + "--includeRecommended", + "--add Microsoft.VisualStudio.Workload.MSBuildTools", + "--add Microsoft.VisualStudio.Workload.VCTools", + ] + + args = [ + "winget", + "install", + "--silent", + "--id", + "Microsoft.VisualStudio.2022.BuildTools", + "--override", + f'"{" ".join(override)}"', + ] + + run( + " ".join(args), + check=False, + ) + + +class WindowsChoco: + """Chocolatey for Windows.""" + + def install(self, command, ci_env): + """Installs packages using Chocolatey.""" + if ci_env: + # don't show noisy choco progress bars in ci env + run(f"{command} --no-progress") + else: + run(command) + + def config_ci_cache(self): + """Configures Chocolatey cache for CI.""" + + runner_temp_key = "RUNNER_TEMP" + runner_temp = os.environ.get(runner_temp_key) + if runner_temp: + # sets the choco cache dir, which should match the dir in the ci cache action. + key_arg = '--name="cacheLocation"' + value_arg = f'--value="{runner_temp}/choco"' + run(["choco", "config", "set", key_arg, value_arg]) + else: + print(f"Warning: CI environment variable {runner_temp_key} not set") + + +class WindowsQt: + """Qt for Windows.""" + + def __init__(self, config): + self.config = config + + self.version = os.environ.get("QT_VERSION") + if not self.version: + try: + default_version = config["version"] + except KeyError: + raise EnvError(f"Qt version not set in {config_file}") + + print(f"QT_VERSION not set, using: {default_version}") + self.version = default_version + + self.base_dir = os.environ.get("QT_BASE_DIR") + if not self.base_dir: + try: + default_base_dir = config["install-dir"] + except KeyError: + raise EnvError(f"Qt install-dir not set in {config_file}") + + print(f"QT_BASE_DIR not set, using: {default_base_dir}") + self.base_dir = default_base_dir + + self.install_dir = f"{self.base_dir}\\{self.version}" + + def get_install_dir(self): + if os.path.isdir(self.install_dir): + return self.install_dir + + def install(self): + """Installs Qt on Windows.""" + + run(["pip", "install", "aqtinstall"]) + + try: + mirror_url = self.config["mirror"] + except KeyError: + raise EnvError(f"Qt mirror not set in {config_file}") + + args = ["python", "-m", "aqt", "install-qt"] + args.extend(["--outputdir", self.base_dir]) + args.extend(["--base", mirror_url]) + args.extend(["windows", "desktop", self.version, "win64_msvc2019_64"]) + run(args) + + install_dir = self.get_install_dir() + if not install_dir: + raise EnvError(f"Qt not installed, path not found: {install_dir}") - runner_temp_key = 'RUNNER_TEMP' - runner_temp = os.environ.get(runner_temp_key) - if runner_temp: - # sets the choco cache dir, which should match the dir in the ci cache action. - key_arg = '--name="cacheLocation"' - value_arg = f'--value="{runner_temp}/choco"' - subprocess.run(['choco', 'config', 'set', key_arg, value_arg], shell=True, check=True) - else: - print(f'Warning: CI environment variable {runner_temp_key} not set') main()