diff --git a/.gitea/workflows/ci.yaml b/.gitea/workflows/ci.yaml index f10f5cf..855f2c7 100644 --- a/.gitea/workflows/ci.yaml +++ b/.gitea/workflows/ci.yaml @@ -7,34 +7,160 @@ on: workflow_dispatch: env: - DOCKER_TLS_CERTDIR: "/certs" - - BASE_BUILDER_TAG: ${{ secrets.CI_REGISTRY_IMAGE }}/linuxbasebuilder:${{ github.ref_name }} - CPP_BUILDER_TAG: ${{ secrets.CI_REGISTRY_IMAGE }}/linuxcppbuilder:${{ github.ref_name }} - BIGFOOT_BUILDER_TAG: ${{ secrets.CI_REGISTRY_IMAGE }}/linuxbigfootbuilder:${{ github.ref_name }} - BIGFOOT_COVERAGE_BUILDER_TAG: ${{ secrets.CI_REGISTRY_IMAGE }}/linuxbigfootcoveragebuilder:${{ github.ref_name }} + BRANCH_NAME: ${{ github.ref_name }} + BASE_BUILDER_TAG: ${{ vars.DOCKER_IMAGE }}/linuxbasebuilder + CPP_BUILDER_TAG: ${{ vars.DOCKER_IMAGE }}/linuxcppbuilder + BIGFOOT_BUILDER_TAG: ${{ vars.DOCKER_IMAGE }}/linuxbigfootbuilder + BIGFOOT_COVERAGE_BUILDER_TAG: ${{ vars.DOCKER_IMAGE }}/linuxbigfootcoveragebuilder jobs: build-base-builder: runs-on: ubuntu-latest timeout-minutes: 360 - container: - image: docker:28.5.1 - options: --privileged - services: - docker: - image: docker:28.5.1-dind - options: --privileged + steps: + - name: Set ENV + run: | + echo "BRANCH_NAME_LOWER=$(echo $GITHUB_REF_NAME | tr '[:upper:]' '[:lower:]')" >> $GITHUB_ENV + echo "BRANCH_NAME=$(echo $GITHUB_REF_NAME)" >> $GITHUB_ENV + + - name: Checkout + uses: actions/checkout@v6 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Login to the Container registry - uses: docker/login-action@v2 + uses: docker/login-action@v3 with: - registry: ${{ secrets.CI_REGISTRY }} - username: ${{ secrets.CI_USERNAME }} - password: ${{ secrets.CI_TOKEN }} + registry: ${{ vars.DOCKER_REGISTRY }} + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_TOKEN }} - # - name: Build Base Builder - # run: docker build -t "$BASE_BUILDER_TAG" ./Linux/BaseBuilder + - name: Build and push base builder + uses: docker/build-push-action@v5 + with: + context: ./Linux/BaseBuilder + push: true + tags: ${{ env.BASE_BUILDER_TAG }}:${{ env.BRANCH_NAME_LOWER }} - # - name: Push Base Builder - # run: docker push "$BASE_BUILDER_TAG" \ No newline at end of file + build-cpp-builder: + runs-on: ubuntu-latest + timeout-minutes: 360 + needs: build-base-builder + + steps: + - name: Set ENV + run: | + echo "BRANCH_NAME_LOWER=$(echo $GITHUB_REF_NAME | tr '[:upper:]' '[:lower:]')" >> $GITHUB_ENV + echo "BRANCH_NAME=$(echo $GITHUB_REF_NAME)" >> $GITHUB_ENV + + - name: Checkout + uses: actions/checkout@v6 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to the Container registry + uses: docker/login-action@v3 + with: + registry: ${{ vars.DOCKER_REGISTRY }} + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_TOKEN }} + + - name: Build and push cpp builder + uses: docker/build-push-action@v5 + with: + context: ./Linux/CPPBuilder + push: true + tags: ${{ env.CPP_BUILDER_TAG }}:${{ env.BRANCH_NAME_LOWER }} + build-args: | + BASE_BUILDER=${{ env.BASE_BUILDER_TAG }}:${{ env.BRANCH_NAME_LOWER }} + CONAN_PROFILE_BRANCH=${{ env.BRANCH_NAME }} + + build-bigfoot-builder: + runs-on: ubuntu-latest + timeout-minutes: 360 + needs: build-cpp-builder + + steps: + - name: Set ENV + run: | + echo "BRANCH_NAME_LOWER=$(echo $GITHUB_REF_NAME | tr '[:upper:]' '[:lower:]')" >> $GITHUB_ENV + echo "BRANCH_NAME=$(echo $GITHUB_REF_NAME)" >> $GITHUB_ENV + + - name: Checkout + uses: actions/checkout@v6 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to the Container registry + uses: docker/login-action@v3 + with: + registry: ${{ vars.DOCKER_REGISTRY }} + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_TOKEN }} + + - name: Build and push bigfoot builder + uses: docker/build-push-action@v5 + with: + context: ./Linux/BigfootBuilder + push: true + tags: ${{ env.BIGFOOT_BUILDER_TAG }}:${{ env.BRANCH_NAME_LOWER }} + build-args: | + BASE_BUILDER=${{ env.CPP_BUILDER_TAG }}:${{ env.BRANCH_NAME_LOWER }} + CONAN_RELEASE_PROFILE=clang + CONAN_RELWITHDEBINFO_PROFILE=clangd + CONAN_DEBUG_PROFILE=clangd + PACKAGES_CHANNEL=${{ env.BRANCH_NAME_LOWER }} + + - name: Push built packages + env: + ARTIFACTORY_USER: ${{ secrets.ARTIFACTORY_USER }} + ARTIFACTORY_PASSWORD: ${{ secrets.ARTIFACTORY_PASSWORD }} + BIGFOOT_BUILDER_TAG: ${{ env.BIGFOOT_BUILDER_TAG }} + BRANCH_NAME_LOWER: ${{ env.BRANCH_NAME_LOWER }} + run: | + docker run --rm \ + -e CONAN_LOGIN_USERNAME=${ARTIFACTORY_USER} \ + -e CONAN_PASSWORD=${ARTIFACTORY_PASSWORD} \ + ${BIGFOOT_BUILDER_TAG}:${BRANCH_NAME_LOWER} \ + conan upload '*' --remote=bigfootpackages --force --confirm + + build-bigfootcoverage-builder: + runs-on: ubuntu-latest + timeout-minutes: 360 + needs: build-cpp-builder + + steps: + - name: Set ENV + run: | + echo "BRANCH_NAME_LOWER=$(echo $GITHUB_REF_NAME | tr '[:upper:]' '[:lower:]')" >> $GITHUB_ENV + echo "BRANCH_NAME=$(echo $GITHUB_REF_NAME)" >> $GITHUB_ENV + + - name: Checkout + uses: actions/checkout@v6 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to the Container registry + uses: docker/login-action@v3 + with: + registry: ${{ vars.DOCKER_REGISTRY }} + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_TOKEN }} + + - name: Build and push bigfootcoverage builder + uses: docker/build-push-action@v5 + with: + context: ./Linux/BigfootBuilder + push: true + tags: ${{ env.BIGFOOT_COVERAGE_BUILDER_TAG }}:${{ env.BRANCH_NAME_LOWER }} + build-args: | + BASE_BUILDER=${{ env.CPP_BUILDER_TAG }}:${{ env.BRANCH_NAME_LOWER }} + CONAN_RELEASE_PROFILE=gcc_coverage + CONAN_RELWITHDEBINFO_PROFILE=gcc_coverage + CONAN_DEBUG_PROFILE=gcc_coverage + PACKAGES_CHANNEL=${{ env.BRANCH_NAME_LOWER }} diff --git a/Linux/BaseBuilder/Dockerfile b/Linux/BaseBuilder/Dockerfile index e3de30b..d30443f 100644 --- a/Linux/BaseBuilder/Dockerfile +++ b/Linux/BaseBuilder/Dockerfile @@ -1,17 +1,24 @@ FROM ubuntu:24.04 -ENV TZ=Europe/Paris -ENV DEBIAN_FRONTEND=noninteractive -ENV PATH="/root/.local/bin:$PATH" +# Set timezone, noninteractive mode, and pipx path +ENV TZ=Europe/Paris \ + DEBIAN_FRONTEND=noninteractive \ + PATH="/root/.local/bin:$PATH" +# Use build cache for ccache RUN --mount=type=cache,target=/ccache -# Make sudo dummy replacement, so we don't weaken docker security -RUN echo "#!/bin/bash\n\$@" > /usr/bin/sudo -RUN chmod +x /usr/bin/sudo +# Dummy sudo (no security weakening) +RUN echo '#!/bin/bash\n$@' > /usr/bin/sudo \ + && chmod +x /usr/bin/sudo -RUN apt-get update -y -RUN apt-get install -y - -RUN apt-get install unzip curl wget lsb-release software-properties-common gnupg -y -RUN apt-get install python3 -y \ No newline at end of file +# Install base dependencies in one layer +RUN apt-get update && apt-get install -y \ + unzip \ + curl \ + wget \ + lsb-release \ + software-properties-common \ + gnupg \ + python3 \ + && rm -rf /var/lib/apt/lists/* diff --git a/Linux/BigfootBuilder/Dockerfile b/Linux/BigfootBuilder/Dockerfile index f8f3f40..1768d4a 100644 --- a/Linux/BigfootBuilder/Dockerfile +++ b/Linux/BigfootBuilder/Dockerfile @@ -5,20 +5,30 @@ FROM $BASE_BUILDER ARG CONAN_RELEASE_PROFILE ARG CONAN_RELWITHDEBINFO_PROFILE ARG CONAN_DEBUG_PROFILE +ARG PACKAGES_CHANNEL -RUN wget -qO - https://packages.lunarg.com/lunarg-signing-key-pub.asc | sudo apt-key add -RUN wget -qO /etc/apt/sources.list.d/lunarg-vulkan-1.4.313-noble.list https://packages.lunarg.com/vulkan/1.4.313/lunarg-vulkan-1.4.313-noble.list -RUN apt-get update -y -RUN apt-get install gcovr -y -RUN apt-get install vulkan-sdk -y -RUN apt-get install xvfb -y +RUN apt-get update \ + && apt-get install -y \ + gcovr \ + xvfb \ + \ + # LunarG Vulkan repo (modern keyring method) + && wget -qO- https://packages.lunarg.com/lunarg-signing-key-pub.asc \ + | gpg --dearmor -o /usr/share/keyrings/lunarg-archive-keyring.gpg \ + \ + && echo "deb [signed-by=/usr/share/keyrings/lunarg-archive-keyring.gpg] https://packages.lunarg.com/vulkan/1.4.313 noble main" \ + > /etc/apt/sources.list.d/lunarg-vulkan.list \ + \ + && apt-get update \ + && apt-get install -y vulkan-sdk \ + \ + # Cleanup + && rm -rf /var/lib/apt/lists/* COPY conanfile.py BigfootDependencies/conanfile.py -RUN ccache --zero-stats - -RUN CONAN_LOGIN_USERNAME=${GITLAB_CONAN_USER} CONAN_PASSWORD=${GITLAB_CONAN_PASSWORD} conan install ./BigfootDependencies --remote=bigfootpackages --build="*" -pr:h=${CONAN_RELEASE_PROFILE} -pr:b=${CONAN_RELEASE_PROFILE} -s build_type=Release && \ - CONAN_LOGIN_USERNAME=${GITLAB_CONAN_USER} CONAN_PASSWORD=${GITLAB_CONAN_PASSWORD} conan install ./BigfootDependencies --remote=bigfootpackages --build="*" -pr:h=${CONAN_RELWITHDEBINFO_PROFILE} -pr:b=${CONAN_RELWITHDEBINFO_PROFILE} -s build_type=RelWithDebInfo && \ - CONAN_LOGIN_USERNAME=${GITLAB_CONAN_USER} CONAN_PASSWORD=${GITLAB_CONAN_PASSWORD} conan install ./BigfootDependencies --remote=bigfootpackages --build="*" -pr:h=${CONAN_DEBUG_PROFILE} -pr:b=${CONAN_DEBUG_PROFILE} -s build_type=Debug - -RUN ccache --show-stats \ No newline at end of file +RUN ccache --zero-stats \ + && conan install ./BigfootDependencies -o Requirements/*:packages_channel=${PACKAGES_CHANNEL} --remote=bigfootpackages --build="*" -pr:h=${CONAN_RELEASE_PROFILE} -pr:b=${CONAN_RELEASE_PROFILE} -s build_type=Release \ + && conan install ./BigfootDependencies -o Requirements/*:packages_channel=${PACKAGES_CHANNEL} --remote=bigfootpackages --build="*" -pr:h=${CONAN_RELWITHDEBINFO_PROFILE} -pr:b=${CONAN_RELWITHDEBINFO_PROFILE} -s build_type=RelWithDebInfo \ + && conan install ./BigfootDependencies -o Requirements/*:packages_channel=${PACKAGES_CHANNEL} --remote=bigfootpackages --build="*" -pr:h=${CONAN_DEBUG_PROFILE} -pr:b=${CONAN_DEBUG_PROFILE} -s build_type=Debug \ + && ccache --show-stats \ No newline at end of file diff --git a/Linux/BigfootBuilder/conanfile.py b/Linux/BigfootBuilder/conanfile.py index a805470..2a43939 100644 --- a/Linux/BigfootBuilder/conanfile.py +++ b/Linux/BigfootBuilder/conanfile.py @@ -3,7 +3,17 @@ from conan import ConanFile class RequirementsConan(ConanFile): name = "Requirements" + options = { + "packages_channel": ["ANY"] + } + default_options = { + "packages_channel": "main" + } + def configure(self): + self.options['mimalloc'].override = True + self.options['mimalloc'].shared = True + self.options['stduuid'].with_cxx20_span = True self.options['flatbuffers'].header_only = True @@ -15,40 +25,38 @@ class RequirementsConan(ConanFile): self.options["benchmark"].enable_lto = True def requirements(self): - self.requires("eastl/3.27.01") - self.requires("unordered_dense/4.8.1") - self.requires("mimalloc/3.1.5") - self.requires("magic_enum/0.9.7") - self.requires("stduuid/1.2.3") - self.requires("sqlite3/3.51.0") + self.requires(f"eastl/3.27.01@bigfootdev/{self.options.packages_channel}") + self.requires(f"unordered_dense/4.8.1@bigfootdev/{self.options.packages_channel}") + self.requires(f"mimalloc/3.1.5@bigfootdev/{self.options.packages_channel}") + self.requires(f"stduuid/1.2.3@bigfootdev/{self.options.packages_channel}") + self.requires(f"sqlite3/3.51.0@bigfootdev/{self.options.packages_channel}") self.requires("cli11/2.6.0") - self.requires("xxhash/0.8.3") + self.requires(f"rapidhash/3.0@bigfootdev/{self.options.packages_channel}") self.requires("effolkronium-random/1.5.0") - self.requires("zeus_expected/1.3.0") - self.requires("flatbuffers/25.9.23") + self.requires(f"flatbuffers/25.12.19@bigfootdev/{self.options.packages_channel}") self.requires("quill/11.0.2") self.requires("tracy/0.12.2") self.requires("cpptrace/1.0.4") self.requires("glm/1.0.1") - self.requires("lodepng/cci.20250727") + self.requires(f"lodepng/cci.20250727@bigfootdev/{self.options.packages_channel}") self.requires("imgui/1.92.5-docking") self.requires("glfw/3.4") self.requires("vulkan-headers/1.4.313.0") - self.requires("vulkan-validationlayers/1.4.313.0") + self.requires(f"vulkan-validationlayers/1.4.313.0@bigfootdev/{self.options.packages_channel}") self.requires("spirv-cross/1.4.313.0") - self.requires("vulkan-memory-allocator/3.3.0") + self.requires(f"vulkan-memory-allocator/3.3.0@bigfootdev/{self.options.packages_channel}") self.requires("gtest/1.17.0") - self.requires("pixelmatch-cpp17/1.0.3") + self.requires(f"pixelmatch-cpp17/1.0.3@bigfootdev/{self.options.packages_channel}") self.requires("stb/cci.20240531", override=True) - self.requires("shaderc/2025.3") + self.requires(f"shaderc/2025.3@bigfootdev/{self.options.packages_channel}") self.requires("assimp/6.0.2") - self.requires("meshoptimizer/1.0") + self.requires(f"meshoptimizer/1.0@bigfootdev/{self.options.packages_channel}") self.requires("libsquish/1.15") self.requires("benchmark/1.9.4") diff --git a/Linux/CPPBuilder/Dockerfile b/Linux/CPPBuilder/Dockerfile index a7b173e..071d5f0 100644 --- a/Linux/CPPBuilder/Dockerfile +++ b/Linux/CPPBuilder/Dockerfile @@ -1,49 +1,73 @@ +# Build arguments ARG BASE_BUILDER +# Base image FROM $BASE_BUILDER -ENV CCACHE_DIR=/ccache -ENV CCACHE_COMPRESS=1 -ENV CCACHE_MAXSIZE=10G -ENV CCACHE_COMPILERCHECK=content +ARG CONAN_PROFILE_BRANCH -RUN apt-get install build-essential -y -RUN apt-get install ccache -y -RUN apt-get install cmake -y -RUN apt-get install ninja-build -y -RUN apt-get install git -y -RUN apt-get install pkg-config -y +# Environment variables for ccache and pipx +ENV CCACHE_DIR=/ccache \ + CCACHE_COMPRESS=1 \ + CCACHE_MAXSIZE=10G \ + CCACHE_COMPILERCHECK=content \ + PATH=/root/.local/bin:$PATH -RUN wget https://apt.llvm.org/llvm.sh && \ - chmod +x llvm.sh && \ - ./llvm.sh 18 all -RUN apt-get install clang-tools-18 -y +# Install system dependencies, clang, and pipx +RUN apt-get update && apt-get install -y \ + build-essential \ + ccache \ + cmake \ + ninja-build \ + git \ + pkg-config \ + wget \ + lsb-release \ + software-properties-common \ + pipx \ + && rm -rf /var/lib/apt/lists/* \ + \ + # Install LLVM 18 + && wget https://apt.llvm.org/llvm.sh -O /tmp/llvm.sh \ + && chmod +x /tmp/llvm.sh \ + && /tmp/llvm.sh 18 all \ + && apt-get install -y clang-tools-18 \ + \ + # Symlinks for clang 18 tools + && ln -sf /usr/bin/clang-18 /usr/bin/clang \ + && ln -sf /usr/bin/clang++-18 /usr/bin/clang++ \ + && ln -sf /usr/bin/llvm-profdata-18 /usr/bin/llvm-profdata \ + && ln -sf /usr/bin/llvm-cov-18 /usr/bin/llvm-cov \ + && ln -sf /usr/bin/clang-format-18 /usr/bin/clang-format \ + && ln -sf /usr/bin/clang-tidy-18 /usr/bin/clang-tidy \ + && ln -sf /usr/bin/llvm-ar-18 /usr/bin/llvm-ar \ + && ln -sf /usr/bin/llvm-nm-18 /usr/bin/llvm-nm \ + && ln -sf /usr/bin/llvm-ranlib-18 /usr/bin/llvm-ranlib \ + \ + # Reset ccache stats + && ccache --zero-stats -RUN ln -s /bin/clang-18 /bin/clang -RUN ln -s /bin/clang++-18 /bin/clang++ -RUN ln -s /usr/bin/llvm-profdata-18 /usr/bin/llvm-profdata -RUN ln -s /usr/bin/llvm-cov-18 /usr/bin/llvm-cov -RUN ln -s /usr/bin/clang-format-18 /usr/bin/clang-format -RUN ln -s /usr/bin/clang-tidy-18 /usr/bin/clang-tidy -RUN ln -s /usr/bin/llvm-ar-18 /usr/bin/llvm-ar -RUN ln -s /usr/bin/llvm-nm-18 /usr/bin/llvm-nm -RUN ln -s /usr/bin/llvm-ranlib-18 /usr/bin/llvm-ranlib +# Install pipx and Conan +RUN pipx ensurepath \ + && pipx install conan -RUN ccache --zero-stats +# Build and install mold +RUN git clone --branch v2.40.4 https://github.com/rui314/mold.git /tmp/mold \ + && cd /tmp/mold \ + && ./install-build-deps.sh \ + && cmake . -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_CXX_COMPILER=clang++ \ + -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ + -DCMAKE_C_COMPILER_LAUNCHER=ccache \ + -B build \ + && cmake --build build -j$(nproc) \ + && cmake --install build \ + && rm -rf /tmp/mold -RUN ccache --show-stats +# Configure Conan with custom profiles and remote +RUN conan config install https://git.romainboullard.com/BigfootDev/ConanProfiles.git --args="--branch $CONAN_PROFILE_BRANCH" \ + && conan remote add bigfootpackages https://conan.romainboullard.com/artifactory/api/conan/BigfootPackages \ + && conan remote disable conancenter -RUN apt-get install pipx -y -RUN pipx ensurepath - -RUN git clone --branch v2.40.4 https://github.com/rui314/mold.git -RUN mold/install-build-deps.sh -RUN cmake mold -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_COMPILER=clang++ -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DCMAKE_C_COMPILER_LAUNCHER=ccache -B mold/build -RUN cmake --build mold/build -j$(nproc) -RUN cmake --build mold/build --target install - -RUN pipx install conan - -RUN conan config install https://gitlab.com/bigfootdev/config/conan.git -RUN conan remote add bigfootpackages https://packages.bigfootengine.com/artifactory/api/conan/bigfootpackages -RUN conan remote disable conancenter \ No newline at end of file +# Show ccache stats (optional, for debugging) +RUN ccache --show-stats \ No newline at end of file