diff --git a/.env.ci.example b/.env.ci.example new file mode 100644 index 0000000..9478583 --- /dev/null +++ b/.env.ci.example @@ -0,0 +1,9 @@ +# Copy to .env.ci and populate real values, then run: +# gh secret set -f .env.ci + +CONAN_REMOTE_URL=https://your-private-conan-remote.example.com +CONAN_LOGIN_USERNAME=your-username +CONAN_PASSWORD=your-password +# Optional fallback names still supported by the workflows: +# GITEA_USERNAME=your-username +# GITEA_TOKEN=your-password diff --git a/.github/scripts/detect_recipe_matrix.py b/.github/scripts/detect_recipe_matrix.py new file mode 100644 index 0000000..7b42df6 --- /dev/null +++ b/.github/scripts/detect_recipe_matrix.py @@ -0,0 +1,164 @@ +#!/usr/bin/env python3 + +import argparse +import json +import os +import re +import subprocess +import sys +from pathlib import Path + + +REPO_ROOT = Path(os.environ.get("GITHUB_WORKSPACE", Path(__file__).resolve().parents[2])).resolve() +RECIPES_DIR = REPO_ROOT / "recipes" +VERSION_LINE_RE = re.compile(r'^ "([^"]+)":\s*$') + + +def run_git(*args, check=True): + try: + result = subprocess.run( + ["git", *args], + cwd=REPO_ROOT, + check=check, + text=True, + capture_output=True, + ) + return result.stdout + except subprocess.CalledProcessError as exc: + raise RuntimeError( + f"git {' '.join(args)} failed in {REPO_ROOT}: {exc.stderr.strip() or exc.stdout.strip() or exc}" + ) from exc + + +def git_file(ref, path): + spec = f"{ref}:{path}" + result = subprocess.run( + ["git", "show", spec], + cwd=REPO_ROOT, + text=True, + capture_output=True, + ) + if result.returncode != 0: + return None + return result.stdout + + +def parse_conandata_versions(text): + if not text: + return {} + + versions = {} + current = None + block = [] + for line in text.splitlines(): + match = VERSION_LINE_RE.match(line) + if match: + if current is not None: + versions[current] = "\n".join(block).rstrip() + current = match.group(1) + block = [line] + elif current is not None: + block.append(line) + if current is not None: + versions[current] = "\n".join(block).rstrip() + return versions + + +def list_versions_from_tree(recipe): + conandata = REPO_ROOT / "recipes" / recipe / "all" / "conandata.yml" + if not conandata.exists(): + return [] + return sorted(parse_conandata_versions(conandata.read_text()).keys()) + + +def list_versions_from_ref(ref, recipe): + path = f"recipes/{recipe}/all/conandata.yml" + return sorted(parse_conandata_versions(git_file(ref, path)).keys()) + + +def compute_changed_versions(base, head, recipe): + path = f"recipes/{recipe}/all/conandata.yml" + before = parse_conandata_versions(git_file(base, path)) + after = parse_conandata_versions(git_file(head, path)) + changed = [] + for version in sorted(set(before) | set(after)): + if before.get(version) != after.get(version): + changed.append(version) + return changed + + +def list_changed_files(base, head): + act_changed_files = os.environ.get("ACT_CHANGED_FILES", "").strip() + if act_changed_files: + return [line.strip() for line in act_changed_files.splitlines() if line.strip()] + return run_git("diff", "--name-only", f"{base}..{head}").splitlines() + + +def build_targets(base, head, mode): + changed_files = list_changed_files(base, head) + targets = {} + + for relpath in changed_files: + parts = Path(relpath).parts + if len(parts) < 2 or parts[0] != "recipes": + continue + + recipe = parts[1] + recipe_bucket = targets.setdefault(recipe, {"all_versions": False, "versions": set()}) + + if len(parts) >= 3 and parts[2] != "all": + recipe_bucket["versions"].add(parts[2]) + continue + + if len(parts) >= 4 and parts[2] == "all" and parts[3] == "conandata.yml": + if mode == "all": + recipe_bucket["all_versions"] = True + else: + recipe_bucket["versions"].update(compute_changed_versions(base, head, recipe)) + continue + + recipe_bucket["all_versions"] = True + + matrix = [] + for recipe, state in sorted(targets.items()): + if state["all_versions"]: + versions = list_versions_from_tree(recipe) or list_versions_from_ref(head, recipe) + else: + versions = sorted(v for v in state["versions"] if v != "all") + + for version in versions: + matrix.append( + { + "name": recipe, + "version": version, + "path": f"recipes/{recipe}/all", + "reference": f"{recipe}/{version}", + } + ) + + return matrix + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--base", required=True) + parser.add_argument("--head", required=True) + parser.add_argument( + "--conandata-mode", + choices=("changed", "all"), + default="changed", + help="How to treat changes to recipes//all/conandata.yml", + ) + args = parser.parse_args() + + matrix = build_targets(args.base, args.head, args.conandata_mode) + payload = {"include": matrix} + print(json.dumps(payload)) + + +if __name__ == "__main__": + try: + main() + except Exception as exc: + print(f"Failed to detect recipe matrix: {exc}", file=sys.stderr) + sys.exit(1) diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml new file mode 100644 index 0000000..77de6a6 --- /dev/null +++ b/.github/workflows/cd.yml @@ -0,0 +1,128 @@ +name: CD + +on: + push: + branches: [main] + paths: + - "recipes/**" + - ".github/workflows/ci.yml" + - ".github/workflows/cd.yml" + - ".github/scripts/**" + workflow_dispatch: + +jobs: + detect-targets: + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.detect.outputs.matrix }} + has_targets: ${{ steps.detect.outputs.has_targets }} + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Detect changed recipe targets + id: detect + shell: bash + run: | + set -euo pipefail + base="${ACT_BASE_SHA:-${{ github.event.before }}}" + if [[ -n "${ACT_CHANGED_FILES:-}" ]]; then + base="${base:-act-base}" + elif [[ -z "$base" || "$base" == "0000000000000000000000000000000000000000" ]]; then + if git rev-parse HEAD^ >/dev/null 2>&1; then + base="$(git rev-parse HEAD^)" + else + base="$(git rev-list --max-parents=0 HEAD)" + fi + fi + head="${ACT_HEAD_SHA:-${{ github.sha }}}" + matrix="$(python3 .github/scripts/detect_recipe_matrix.py --base "$base" --head "$head" --conandata-mode changed)" + echo "matrix=$matrix" >> "$GITHUB_OUTPUT" + if [[ "$matrix" == '{"include": []}' ]]; then + echo "has_targets=false" >> "$GITHUB_OUTPUT" + else + echo "has_targets=true" >> "$GITHUB_OUTPUT" + fi + + publish: + needs: detect-targets + if: needs.detect-targets.outputs.has_targets == 'true' + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: ${{ fromJson(needs.detect-targets.outputs.matrix) }} + env: + CONAN_REMOTE_URL: ${{ secrets.CONAN_REMOTE_URL }} + CONAN_LOGIN_USERNAME: ${{ secrets.CONAN_LOGIN_USERNAME || secrets.GITEA_USERNAME }} + CONAN_PASSWORD: ${{ secrets.CONAN_PASSWORD || secrets.GITEA_TOKEN }} + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Install host build tools + run: | + sudo apt-get update + sudo apt-get install -y --no-install-recommends \ + build-essential \ + gcc-14 \ + g++-14 \ + cmake \ + ninja-build \ + graphviz \ + crudini + + - name: Set gcc-14 as default + run: | + sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-14 10 + sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-14 10 + + - name: Setup Conan client + uses: conan-io/setup-conan@v1 + with: + version: "2.*" + use_venv: true + + - name: Validate required Conan secrets + run: | + test -n "$CONAN_REMOTE_URL" || (echo "Missing secret: CONAN_REMOTE_URL" && exit 1) + test -n "$CONAN_LOGIN_USERNAME" || (echo "Missing secret: CONAN_LOGIN_USERNAME" && exit 1) + test -n "$CONAN_PASSWORD" || (echo "Missing secret: CONAN_PASSWORD" && exit 1) + + - name: Detect Conan profile + run: | + conan profile detect --force + crudini --set ~/.conan2/profiles/default settings compiler.cppstd 23 + + - name: Login private Conan remote + run: | + conan remote add org "$CONAN_REMOTE_URL" --force + conan remote login org "$CONAN_LOGIN_USERNAME" --password "$CONAN_PASSWORD" + conan remote list + + - name: Build package + run: | + conan create "${{ matrix.path }}" --version="${{ matrix.version }}" --build=missing + + - name: Upload package if missing on remote + shell: bash + run: | + set -euo pipefail + ref="${{ matrix.reference }}" + remote_json="$(conan list "${ref}:*" -r org --format=json || true)" + if python3 -c 'import json, sys; data = json.loads((sys.argv[1].strip() or "{}")); has_package = any(isinstance(recipe_data, dict) and recipe_data.get("packages") for remote_data in data.values() if isinstance(remote_data, dict) for recipe_data in remote_data.values()); sys.exit(0 if has_package else 1)' "$remote_json" + then + echo "Remote already has packages for ${ref}; skipping upload." + exit 0 + fi + + conan upload "${ref}" -r org --confirm + + no-targets: + needs: detect-targets + if: needs.detect-targets.outputs.has_targets != 'true' + runs-on: ubuntu-latest + steps: + - name: No changed recipes + run: echo "No recipe targets detected for publishing." diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..bc269cd --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,99 @@ +name: CI + +on: + pull_request: + branches: [main] + paths: + - "recipes/**" + - ".github/workflows/ci.yml" + - ".github/workflows/cd.yml" + - ".github/scripts/**" + workflow_dispatch: + +jobs: + detect-targets: + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.detect.outputs.matrix }} + has_targets: ${{ steps.detect.outputs.has_targets }} + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Detect changed recipe targets + id: detect + shell: bash + run: | + set -euo pipefail + if [[ -n "${ACT_CHANGED_FILES:-}" ]]; then + base="act-base" + elif [[ "${{ github.event_name }}" == "pull_request" ]]; then + base="${{ github.event.pull_request.base.sha }}" + else + if git rev-parse HEAD^ >/dev/null 2>&1; then + base="$(git rev-parse HEAD^)" + else + base="$(git rev-list --max-parents=0 HEAD)" + fi + fi + head="${ACT_HEAD_SHA:-${{ github.sha }}}" + matrix="$(python3 .github/scripts/detect_recipe_matrix.py --base "$base" --head "$head" --conandata-mode changed)" + echo "matrix=$matrix" >> "$GITHUB_OUTPUT" + if [[ "$matrix" == '{"include": []}' ]]; then + echo "has_targets=false" >> "$GITHUB_OUTPUT" + else + echo "has_targets=true" >> "$GITHUB_OUTPUT" + fi + + conan-create: + needs: detect-targets + if: needs.detect-targets.outputs.has_targets == 'true' + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: ${{ fromJson(needs.detect-targets.outputs.matrix) }} + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Install host build tools + run: | + sudo apt-get update + sudo apt-get install -y --no-install-recommends \ + build-essential \ + gcc-14 \ + g++-14 \ + cmake \ + ninja-build \ + graphviz \ + crudini + + - name: Set gcc-14 as default + run: | + sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-14 10 + sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-14 10 + + - name: Setup Conan client + uses: conan-io/setup-conan@v1 + with: + version: "2.*" + use_venv: true + + - name: Detect Conan profile + run: | + conan profile detect --force + crudini --set ~/.conan2/profiles/default settings compiler.cppstd 23 + + - name: Build and test package + run: | + conan create "${{ matrix.path }}" --version="${{ matrix.version }}" --build=missing + + no-targets: + needs: detect-targets + if: needs.detect-targets.outputs.has_targets != 'true' + runs-on: ubuntu-latest + steps: + - name: No changed recipes + run: echo "No recipe targets detected for validation." diff --git a/.gitignore b/.gitignore index 158fd1f..1ffbdd5 100644 --- a/.gitignore +++ b/.gitignore @@ -140,4 +140,13 @@ venv.bak/ .sw? Session.vim *~ -.undodir \ No newline at end of file +.undodir + +/.* +!/.gitignore +!/.editorconfig +!/.gitattributes +!/.github +!/.vscode +/.env* +!/.env*.example diff --git a/README.md b/README.md index e7e98a9..599e3eb 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,15 @@ The goal of this repo is provide some **non-official** recipes for libraries that has no plan to officially support conan. +## CI/CD + +* `ci.yml` validates only recipe/version targets changed by a pull request. +* `cd.yml` runs on `main`, builds only changed recipe/version targets, and uploads only missing packages to the configured private Conan remote. +* Populate secrets from `.env.ci.example` or `.env.ci` with `gh secret set -f .env.ci`. + +## ref + * bibliography: [conan-center-index](https://github.com/conan-io/conan-center-index/tree/master) * [audacity/conan-recipes](https://github.com/audacity/conan-recipes) * [bkinnightskytw/ethercat](https://gitlab.com/bkinnightskytw/ethercat/-/tree/backport/1.5.3-conan?ref_type=heads) +