diff --git a/build/jenkins/pipelines/bundles.Jenkinsfile b/build/jenkins/pipelines/bundles.Jenkinsfile index 49c7505a86..192791e3c0 100644 --- a/build/jenkins/pipelines/bundles.Jenkinsfile +++ b/build/jenkins/pipelines/bundles.Jenkinsfile @@ -1,4 +1,4 @@ -/* Copyright (C) 2023 Wildfire Games. +/* Copyright (C) 2024 Wildfire Games. * This file is part of 0 A.D. * * 0 A.D. is free software: you can redistribute it and/or modify @@ -15,89 +15,88 @@ * along with 0 A.D. If not, see . */ +// This pipeline is used to generate bundles (Windows installer, macOS package, and source tarballs). + pipeline { - agent { label 'MacSlave' } + agent { + node { + label 'macOSAgentVentura' + } + } // Archive the installer for public download; keep only the latest one. options { - buildDiscarder(logRotator(artifactNumToKeepStr: '1')) + buildDiscarder logRotator(artifactNumToKeepStr: '1') + skipDefaultCheckout true } parameters { - string(name: 'BUNDLE_VERSION', defaultValue: '0.0.27dev', description: 'Bundle Version') - string(name: 'ENGINE_VERSION', defaultValue: '0.0.27', description: 'Engine Version') - string(name: 'SVN_REV', defaultValue: 'HEAD', description: 'For instance 21000') - booleanParam(name: 'ONLY_MOD', defaultValue: true, description: 'Only archive the mod mod.') + string(name: 'BUNDLE_VERSION', defaultValue: '0.27.0dev', description: 'Bundle Version') booleanParam(name: 'DO_GZIP', defaultValue: true, description: 'Create .gz unix tarballs as well as .xz') - booleanParam(name: 'FULL_REBUILD', defaultValue: true, description: 'Do a full rebuild (safer for release, slower).') - booleanParam(name: 'WINDOWS_UNIX', defaultValue: true, description: 'Build windows and unix bundles.') - booleanParam(name: 'BUILD_SHADERS', defaultValue: true, description: 'Build the spir-v shaders (very slow).') + } + + environment { + MIN_OSX_VERSION = "10.12" } stages { - stage ("Checkout") { - options { - retry(3) - } + stage("Checkout Nightly Build") { steps { - script { - try { - sh "svn co https://svn.wildfiregames.com/public/ps/trunk@${params.SVN_REV} ." - } catch(e) { - sh "svn cleanup" - sleep 300 - throw e - } - sh "svn cleanup" - // Delete unknown files everywhere except libraries/ - sh "svn st --no-ignore . --depth immediates | cut -c 9- | xargs rm -rfv" - sh "svn st --no-ignore {binaries/,build/,source/} | cut -c 9- | xargs rm -rfv" - if (params.FULL_REBUILD) { - // Also delete libraries/ - sh "svn st --no-ignore | cut -c 9- | xargs rm -rfv" - } - sh "svn revert . -R" - } + checkout changelog: false, poll: false, scm: [ + $class: 'SubversionSCM', + locations: [[local: '.', remote: 'https://svn.wildfiregames.com/nightly-build/trunk']], + quietOperation: false, + workspaceUpdater: [$class: 'UpdateWithCleanUpdater']] + sh "svn cleanup" } } - stage("Compile Mac Executable") { + + stage("Compile Native macOS Executable") { steps { - sh "JOBS=\"-j\$(sysctl -n hw.ncpu)\" FULL_REBUILD=${params.FULL_REBUILD} source/tools/dist/build-osx-executable.sh" + sh "cd libraries/ && MIN_OSX_VERSION=${env.MIN_OSX_VERSION} ./build-macos-libs.sh --force-rebuild" + sh "cd build/workspaces/ && ./update-workspaces.sh --macosx-version-min=${env.MIN_OSX_VERSION}" + sh "cd build/workspaces/gcc/ && make" + sh "svn cleanup --remove-unversioned build" + sh "svn cleanup --remove-unversioned libraries" } } - stage("Create archive data") { + + stage("Create Mod Archives") { steps { - sh "JOBS=\"-j\$(sysctl -n hw.ncpu)\" ONLY_MOD=${params.ONLY_MOD} BUILD_SHADERS=${params.BUILD_SHADERS} ENGINE_VERSION=${params.ENGINE_VERSION} source/tools/dist/build-archives.sh" + sh "source/tools/dist/build-archives.sh" } } - stage("Create Mac Bundle") { + + stage("Create Native macOS Bundle") { steps { - sh "python3 source/tools/dist/build-osx-bundle.py ${params.BUNDLE_VERSION}" + sh "/opt/wfg/venv/bin/python3 source/tools/dist/build-osx-bundle.py --min_osx=${env.MIN_OSX_VERSION} ${params.BUNDLE_VERSION}" } } - stage("Create Windows installer & *nix files") { + + stage("Compile Intel macOS Executable") { + environment { + ARCH = "x86_64" + HOSTTYPE = "x86_64" + } steps { - script { - if(params.WINDOWS_UNIX) - { - // The files created by the mac compilation need to be deleted - sh "svn st {binaries/,build/} --no-ignore | cut -c 9- | xargs rm -rfv" - // Hide the libraries folder. - sh "mv libraries/ temp_libraries/" - sh "svn revert libraries/ -R" - // The generated tests use hardcoded paths so they must be deleted as well. - sh 'python3 -c \"import glob; print(\\\" \\\".join(glob.glob(\\\"source/**/tests/**.cpp\\\", recursive=True)));\" | xargs rm -v' - sh "svn revert build/ -R" - try { - // Then run the core object. - sh "JOBS=\"-j\$(sysctl -n hw.ncpu)\" BUNDLE_VERSION=${params.BUNDLE_VERSION} DO_GZIP=${params.DO_GZIP} source/tools/dist/build-unix-win32.sh" - } finally { - // Un-hide the libraries. - sh "rm -rfv libraries/" - sh "mv temp_libraries/ libraries/" - } - } - } + sh "cd libraries/ && MIN_OSX_VERSION=${env.MIN_OSX_VERSION} ./build-macos-libs.sh --force-rebuild" + sh "cd build/workspaces/ && ./update-workspaces.sh --macosx-version-min=${env.MIN_OSX_VERSION}" + sh "cd build/workspaces/gcc/ && make clean" + sh "cd build/workspaces/gcc/ && make" + sh "svn cleanup --remove-unversioned build" + sh "svn cleanup --remove-unversioned libraries" + } + } + + stage("Create Intel macOS Bundle") { + steps { + sh "/opt/wfg/venv/bin/python3 source/tools/dist/build-osx-bundle.py --architecture=x86_64 --min_osx=${env.MIN_OSX_VERSION} ${params.BUNDLE_VERSION}" + } + } + + stage("Create Windows Installer & Tarballs") { + steps { + sh "BUNDLE_VERSION=${params.BUNDLE_VERSION} DO_GZIP=${params.DO_GZIP} source/tools/dist/build-unix-win32.sh" } } } @@ -105,6 +104,7 @@ pipeline { post { success { archiveArtifacts '*.dmg,*.exe,*.tar.gz,*.tar.xz' + sh "shasum -a 1 *.{dmg,exe,tar.gz,tar.xz}" } } } diff --git a/build/premake/premake5.lua b/build/premake/premake5.lua index b6fc3f381f..b71483449a 100644 --- a/build/premake/premake5.lua +++ b/build/premake/premake5.lua @@ -94,6 +94,7 @@ else f:close() end -- Special handling on mac os where xcode needs special flags. + -- TODO: We should look into "universal" macOS compilation. if os.istarget("macosx") then if string.find(machine, "arm64") then arch = "aarch64" diff --git a/source/tools/dist/0ad.nsi b/source/tools/dist/0ad.nsi index 8ffaddbc10..f25ab821e1 100644 --- a/source/tools/dist/0ad.nsi +++ b/source/tools/dist/0ad.nsi @@ -1,6 +1,7 @@ ; To generate the installer (on Linux): -; Do an 'svn export' into a directory called e.g. "export-win32" -; makensis -nocd -dcheckoutpath=export-win32 -dversion=0.1.2 -dprefix=0ad-0.1.2-alpha export-win32/source/tools/dist/0ad.nsi +; Export the nightly build into a directory called e.g. "nightly-build" +; Archivebuild the mod and public mods into e.g. "archives" +; makensis -nocd -dcheckoutpath=nightly-build -dversion=0.xxx.0 -dprefix=0ad-0.xxx.0 -darchive_path=archives nightly-build/source/tools/dist/0ad.nsi SetCompressor /SOLID LZMA diff --git a/source/tools/dist/build-archives.sh b/source/tools/dist/build-archives.sh index 66c06dd3f1..86ffb9b413 100755 --- a/source/tools/dist/build-archives.sh +++ b/source/tools/dist/build-archives.sh @@ -8,7 +8,7 @@ die() } # Build the mod .zip using the pyrogenesis executable. -# Assumes it is being run from trunk/ +# This must be run from the trunk of a nightly build (containing all translations and SPIR-V shaders) echo "Building archives" @@ -38,48 +38,6 @@ else fi cd - || die -BUILD_SHADERS="${BUILD_SHADERS:=true}" -if [ "${BUILD_SHADERS}" = true ]; then - PYTHON=${PYTHON:=$(command -v python3 || command -v python || true)} - GLSLC=${GLSLC:=$(command -v glslc || true)} - SPIRV_REFLECT=${SPIRV_REFLECT:=$(command -v spirv-reflect || true)} - if [ -e "$(realpath libraries/source/spirv-reflect/bin/spirv-reflect || true)" ]; then - : "${SPIRV_REFLECT:=$(realpath libraries/source/spirv-reflect/bin/spirv-reflect || true)}" - fi - export SPIRV_REFLECT - - [ -n "${PYTHON}" ] || die "Error: python is not available. Install it before proceeding." - [ -n "${GLSLC}" ] || - die "Error: glslc is not available." \ - " Install it with the Vulkan SDK or shaderc package before proceeding." - [ -n "${SPIRV_REFLECT}" ] || - die "Error: spirv-reflect is not available." \ - " Install it with the Vulkan SDK or build vendored spirv-reflect before proceeding." - - cd source/tools/spirv || die - - ENGINE_VERSION=${ENGINE_VERSION:="0.0.xx"} - rulesFile="rules.${ENGINE_VERSION}.json" - - if [ ! -e "$rulesFile" ]; then - # The rules.json file should be present in release tarballs, for - # some Linux CIs don't have access to the internet. - download="$(command -v wget || echo "curl -sLo ""${rulesFile}""")" - $download "https://releases.wildfiregames.com/spir-v/$rulesFile" - fi - - for modname in $archives; do - modLocation="../../../binaries/data/mods/${modname}" - if [ -e "${modLocation}/shaders/spirv/" ]; then - echo "Removing existing spirv shaders for '${modname}'..." - rm -rf "${modLocation}/shaders/spirv" - fi - echo "Building shader for '${modname}'..." - $PYTHON compile.py "$modLocation" "$rulesFile" "$modLocation" --dependency "../../../binaries/data/mods/mod/" - done - cd - || die -fi - for modname in $archives; do echo "Building archive for '${modname}'..." ARCHIVEBUILD_INPUT="binaries/data/mods/${modname}" diff --git a/source/tools/dist/build-osx-bundle.py b/source/tools/dist/build-osx-bundle.py index 6fda6f9553..9181562787 100755 --- a/source/tools/dist/build-osx-bundle.py +++ b/source/tools/dist/build-osx-bundle.py @@ -20,6 +20,8 @@ import dmgbuild parser = argparse.ArgumentParser() parser.add_argument('bundle_version', help='Bundle version') +parser.add_argument('--architecture', help='aarch64 (arm64) or x86_64 (amd64)', + default='aarch64') parser.add_argument('--min_osx', help='Minimum supported OSX version', default='10.12') parser.add_argument('--bundle_identifier', help='Bundle identifier', @@ -28,11 +30,12 @@ parser.add_argument('--dev', help='Turn on dev mode, which isn\'t fit for releas action="store_true") args = parser.parse_args() +ARCH = args.architecture BUNDLE_IDENTIFIER = args.bundle_identifier BUNDLE_VERSION = args.bundle_version BUNDLE_MIN_OSX_VERSION = args.min_osx -BUNDLE_DMG_NAME = f"0ad-{BUNDLE_VERSION}-alpha-osx64" +BUNDLE_DMG_NAME = f"0ad-{BUNDLE_VERSION}-macos-{ARCH}" BUNDLE_OUTPUT = "0 A.D..app" BUNDLE_CONTENTS = BUNDLE_OUTPUT + "/Contents" BUNDLE_BIN = BUNDLE_CONTENTS + "/MacOS" diff --git a/source/tools/dist/build-osx-executable.sh b/source/tools/dist/build-osx-executable.sh deleted file mode 100755 index e89f61aa81..0000000000 --- a/source/tools/dist/build-osx-executable.sh +++ /dev/null @@ -1,77 +0,0 @@ -#!/bin/sh - -# Build the Pyrogenesis executable, used to create the bundle and run the archiver. - -export ARCH="${ARCH:=$(uname -m)}" - -# Set mimimum required OS X version, SDK location and tools -# Old SDKs can be found at https://github.com/phracker/MacOSX-SDKs -export MIN_OSX_VERSION="${MIN_OSX_VERSION:="10.12"}" -# Note that the 10.12 SDK is know to be too old for FMT 7. -export SYSROOT="${SYSROOT:="/Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk"}" -export CC="${CC:="clang"}" CXX="${CXX:="clang++"}" - -die() -{ - echo ERROR: "$*" - exit 1 -} - -# Check that we're actually on OS X -if [ "$(uname -s)" != "Darwin" ]; then - die "This script is intended for OS X only" -fi - -# Check SDK exists -if [ ! -d "${SYSROOT}" ]; then - die "${SYSROOT} does not exist! You probably need to install Xcode" -fi - -cd "build/workspaces/" || die - -JOBS=${JOBS:="-j5"} - -# Toggle whether this is a full rebuild, including libraries (takes much longer). -FULL_REBUILD=${FULL_REBUILD:=false} - -if [ "$FULL_REBUILD" = true ]; then - CLEAN_WORKSPACE_ARGS="" - BUILD_LIBS_ARGS="--force-rebuild" -else - CLEAN_WORKSPACE_ARGS="--preserve-libs" - BUILD_LIBS_ARGS="" -fi - -./clean-workspaces.sh "${CLEAN_WORKSPACE_ARGS}" - -# Build libraries against SDK -echo -echo "Building libraries" -echo -cd ../../libraries/osx || die -SYSROOT="${SYSROOT}" MIN_OSX_VERSION="${MIN_OSX_VERSION}" \ - ./build-osx-libs.sh "$JOBS" "${BUILD_LIBS_ARGS}" || die "Libraries build script failed" -cd - || die - -# Update workspaces -echo -echo "Generating workspaces" -echo - -# Pass OS X options through to Premake -(./update-workspaces.sh --sysroot="${SYSROOT}" --macosx-version-min="${MIN_OSX_VERSION}") || die "update-workspaces.sh failed!" - -echo -echo "Building game" -echo -cd gcc || die -(make clean && CC="$CC -arch $ARCH" CXX="$CXX -arch $ARCH" make "${JOBS}") || die "Game build failed!" -cd - || die - -# Run test to confirm all is OK -echo -echo "Running tests" -echo -cd ../../binaries/system || die -./test || die "Post-build testing failed!" -cd - || die diff --git a/source/tools/dist/build-unix-win32.sh b/source/tools/dist/build-unix-win32.sh index 8594b606bf..e8b2a9aff6 100755 --- a/source/tools/dist/build-unix-win32.sh +++ b/source/tools/dist/build-unix-win32.sh @@ -4,18 +4,15 @@ set -ev XZOPTS="-9 -e" GZIP7ZOPTS="-mx=9" -BUNDLE_VERSION=${BUNDLE_VERSION:="0.0.xxx"} -PREFIX="0ad-${BUNDLE_VERSION}-alpha" +BUNDLE_VERSION=${BUNDLE_VERSION:="0.27.0dev"} +PREFIX="0ad-${BUNDLE_VERSION}" # Collect the relevant files tar cf "$PREFIX"-unix-build.tar \ - --exclude='*.bat' --exclude='*.dll' --exclude='*.exe' --exclude='*.lib' \ - --exclude='libraries/source/fcollada/src/FCollada/FColladaTest' \ - --exclude='libraries/source/spidermonkey/include-*' \ - --exclude='libraries/source/spidermonkey/lib*' \ - --exclude='source/test_root.cpp' \ + --exclude='*.bat' --exclude='*.dll' --exclude='*.exe' --exclude='*.lib' --exclude='*.pdb' \ + --exclude='libraries/win32' \ -s "|.|$PREFIX/~|" \ - source build libraries/source binaries/system/readme.txt binaries/data/l10n binaries/data/tests binaries/data/mods/_test.* ./*.txt + source build libraries binaries/system/readme.txt binaries/data/l10n binaries/data/tests binaries/data/mods/_test.* ./*.txt tar cf "$PREFIX"-unix-data.tar \ --exclude='binaries/data/config/dev.cfg' \ @@ -31,13 +28,20 @@ xz -kv ${XZOPTS} "$PREFIX"-unix-build.tar xz -kv ${XZOPTS} "$PREFIX"-unix-data.tar DO_GZIP=${DO_GZIP:=true} if [ "$DO_GZIP" = true ]; then - 7z a ${GZIP7ZOPTS} "$PREFIX"-unix-build.tar.gz "$PREFIX"-unix-build.tar - 7z a ${GZIP7ZOPTS} "$PREFIX"-unix-data.tar.gz "$PREFIX"-unix-data.tar + 7zz a ${GZIP7ZOPTS} "$PREFIX"-unix-build.tar.gz "$PREFIX"-unix-build.tar + 7zz a ${GZIP7ZOPTS} "$PREFIX"-unix-data.tar.gz "$PREFIX"-unix-data.tar fi # Create Windows installer # This needs nsisbi for files > 2GB -makensis -V4 -nocd \ +# nsisbi 3.09 and above fail to build on macOS, nsisbi 3.08.1 is used on the CD +# To build and install on macOS: +# - install mingw-w64 and scons with Homebrew +# - download the latest source at https://sourceforge.net/projects/nsisbi/files/ +# - build with `scons SKIPUTILS="Makensisw","NSIS Menu","zip2exe"` +# - install with `sudo scons install SKIPUTILS="Makensisw","NSIS Menu","zip2exe"` +# Running makensis also needs a LANG workaround for https://sourceforge.net/p/nsis/bugs/1165/ +LANG="en_GB.UTF-8" makensis -V4 -nocd \ -dcheckoutpath="." \ -dversion="${BUNDLE_VERSION}" \ -dprefix="${PREFIX}" \ @@ -48,15 +52,7 @@ makensis -V4 -nocd \ chmod -f 644 "${PREFIX}-unix-build.tar.xz" chmod -f 644 "${PREFIX}-unix-data.tar.xz" chmod -f 644 "${PREFIX}-win32.exe" - -# Print digests for copying into wiki page -shasum -a 1 "${PREFIX}-unix-build.tar.xz" -shasum -a 1 "${PREFIX}-unix-data.tar.xz" -shasum -a 1 "${PREFIX}-win32.exe" - if [ "$DO_GZIP" = true ]; then chmod -f 644 "${PREFIX}-unix-build.tar.gz" chmod -f 644 "${PREFIX}-unix-data.tar.gz" - shasum -a 1 "${PREFIX}-unix-build.tar.gz" - shasum -a 1 "${PREFIX}-unix-data.tar.gz" fi diff --git a/source/tools/dist/build.sh b/source/tools/dist/build.sh deleted file mode 100755 index 5358b32ab0..0000000000 --- a/source/tools/dist/build.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/sh -set -e - -## This script runs all necessary steps to make a bundle -## It is not used directly by the CI, which calls those steps independently. -## Assume we are being called from trunk/ - -./source/tools/dist/build-osx-executable.sh -./source/tools/dist/build-archives.sh -python3 source/tools/dist/build-osx-bundle.py 0ad-dev -# Note that at this point, you'll have left-over compilation files. -# The CI cleans them via svn st | cut -c9- | xargs rm -rf -BUNDLE_VERSION=0ad-dev ./source/tools/dist/build-unix-win32.sh