diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5c57a1b..74997de 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -54,6 +54,37 @@ jobs: - name: Test API TLS run: scripts/test-api-tls.sh + container: + name: Container (${{ matrix.name }}) + if: github.event_name == 'pull_request' + runs-on: ${{ matrix.runner }} + strategy: + fail-fast: false + matrix: + include: + - name: Linux AMD64 + runner: ubuntu-24.04 + - name: Linux ARM64 + runner: ubuntu-24.04-arm + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Install Nix + uses: cachix/install-nix-action@v31 + + - name: Enable Magic Nix Cache + uses: DeterminateSystems/magic-nix-cache-action@v14 + + - name: Build container + run: nix build .#container + + - name: Test container + run: | + docker load --input result + docker image inspect upgrid:nix + docker run --rm --privileged upgrid:nix --help + frontend: name: Frontend runs-on: ubuntu-latest diff --git a/.github/workflows/publish-container.yml b/.github/workflows/publish-container.yml index a46ef24..fe6fc0c 100644 --- a/.github/workflows/publish-container.yml +++ b/.github/workflows/publish-container.yml @@ -30,45 +30,64 @@ jobs: include: - name: Linux AMD64 runner: ubuntu-24.04 - platform: linux/amd64 artifact: linux-amd64 - name: Linux ARM64 runner: ubuntu-24.04-arm - platform: linux/arm64 artifact: linux-arm64 steps: - name: Checkout uses: actions/checkout@v4 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 + - name: Install Nix + uses: cachix/install-nix-action@v31 - - name: Log in to GHCR - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} + - name: Enable Magic Nix Cache + uses: DeterminateSystems/magic-nix-cache-action@v14 + + - name: Build binary + run: nix build .#upgrid --out-link result-upgrid + + - name: Package release binary + if: github.ref_type == 'tag' + shell: bash + run: | + archive="upgrid-${GITHUB_REF_NAME}-${{ matrix.artifact }}.tar.gz" + tar --create --gzip --file "$archive" --directory result-upgrid/bin upgrid - - name: Build and publish digest - id: build - uses: docker/build-push-action@v6 + - name: Upload release binary + if: github.ref_type == 'tag' + uses: actions/upload-artifact@v4 with: - context: . - target: runtime - platforms: ${{ matrix.platform }} - outputs: type=image,name=${{ env.IMAGE_NAME }},push-by-digest=true,name-canonical=true,push=true - cache-from: type=gha,scope=container-${{ matrix.artifact }} - cache-to: type=gha,mode=max,scope=container-${{ matrix.artifact }} - - - name: Export digest + name: upgrid-${{ matrix.artifact }} + path: upgrid-*.tar.gz + if-no-files-found: error + + - name: Build container + run: nix build .#container + + - name: Install Skopeo + run: nix profile install nixpkgs#skopeo + + - name: Publish architecture image env: - DIGEST: ${{ steps.build.outputs.digest }} + ARTIFACT: ${{ matrix.artifact }} + GHCR_TOKEN: ${{ secrets.GITHUB_TOKEN }} shell: bash run: | + image="${IMAGE_NAME}:build-${GITHUB_SHA}-${ARTIFACT}" mkdir -p /tmp/digests - touch "/tmp/digests/${DIGEST#sha256:}" + skopeo copy \ + --dest-registry-token "${GHCR_TOKEN}" \ + --digestfile /tmp/digest \ + "docker-archive:result" \ + "docker://${image}" + digest="$(cat /tmp/digest)" + if [[ ! "$digest" =~ ^sha256:[0-9a-f]{64}$ ]]; then + echo "could not read published image digest" >&2 + exit 1 + fi + touch "/tmp/digests/${digest#sha256:}" - name: Upload digest uses: actions/upload-artifact@v4 @@ -104,24 +123,30 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Select image tag + - name: Select image tags id: image shell: bash run: | if [[ "$GITHUB_REF_TYPE" == "tag" ]]; then - tag="$GITHUB_REF_NAME" + primary_tag="$GITHUB_REF_NAME" + moving_tag="latest" else - tag="main-${GITHUB_SHA:0:6}" + primary_tag="main-${GITHUB_SHA:0:6}" + moving_tag="latest-unstable" fi - if [[ ! "$tag" =~ ^[A-Za-z0-9_][A-Za-z0-9_.-]{0,127}$ ]]; then - echo "invalid container tag: $tag" >&2 - exit 1 - fi - echo "tag=$tag" >> "$GITHUB_OUTPUT" + for tag in "$primary_tag" "$moving_tag"; do + if [[ ! "$tag" =~ ^[A-Za-z0-9_][A-Za-z0-9_.-]{0,127}$ ]]; then + echo "invalid container tag: $tag" >&2 + exit 1 + fi + done + echo "primary_tag=$primary_tag" >> "$GITHUB_OUTPUT" + echo "moving_tag=$moving_tag" >> "$GITHUB_OUTPUT" - name: Create multi-architecture image env: - IMAGE_TAG: ${{ steps.image.outputs.tag }} + PRIMARY_TAG: ${{ steps.image.outputs.primary_tag }} + MOVING_TAG: ${{ steps.image.outputs.moving_tag }} shell: bash working-directory: /tmp/digests run: | @@ -130,84 +155,49 @@ jobs: images+=("${IMAGE_NAME}@sha256:${digest}") done docker buildx imagetools create \ - --tag "${IMAGE_NAME}:${IMAGE_TAG}" \ + --tag "${IMAGE_NAME}:${PRIMARY_TAG}" \ + --tag "${IMAGE_NAME}:${MOVING_TAG}" \ "${images[@]}" - docker buildx imagetools inspect "${IMAGE_NAME}:${IMAGE_TAG}" + docker buildx imagetools inspect "${IMAGE_NAME}:${PRIMARY_TAG}" + docker buildx imagetools inspect "${IMAGE_NAME}:${MOVING_TAG}" build-release-binaries: name: Build ${{ matrix.name }} binary if: github.ref_type == 'tag' - needs: publish + needs: build-container runs-on: ${{ matrix.runner }} - permissions: - contents: read - packages: read strategy: fail-fast: false matrix: include: - - name: Linux AMD64 - runner: ubuntu-24.04 - artifact: linux-amd64 - build: container - - name: Linux ARM64 - runner: ubuntu-24.04-arm - artifact: linux-arm64 - build: container - name: macOS AMD64 runner: macos-15-intel artifact: macos-amd64 - build: cargo target: x86_64-apple-darwin - name: macOS ARM64 runner: macos-15 artifact: macos-arm64 - build: cargo target: aarch64-apple-darwin - name: Windows AMD64 runner: windows-latest artifact: windows-amd64 - build: cargo target: x86_64-pc-windows-msvc steps: - name: Checkout - if: matrix.build == 'cargo' uses: actions/checkout@v4 - name: Install Rust - if: matrix.build == 'cargo' uses: dtolnay/rust-toolchain@nightly with: targets: ${{ matrix.target }} - name: Cache Rust build - if: matrix.build == 'cargo' uses: Swatinem/rust-cache@v2 with: key: release-${{ matrix.target }} - - name: Log in to GHCR - if: matrix.build == 'container' - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Extract binary from image - if: matrix.build == 'container' - shell: bash - run: | - image="${IMAGE_NAME}:${GITHUB_REF_NAME}" - docker pull "$image" - container_id="$(docker create "$image")" - trap 'docker rm --force "$container_id"' EXIT - mkdir output - docker cp "$container_id:/usr/local/bin/upgrid" output/upgrid - - name: Compile binary - if: matrix.build == 'cargo' run: cargo build --locked --release -p upgrid --target ${{ matrix.target }} - name: Package Unix binary @@ -215,10 +205,8 @@ jobs: shell: bash run: | archive="upgrid-${GITHUB_REF_NAME}-${{ matrix.artifact }}.tar.gz" - if [[ "${{ matrix.build }}" == "cargo" ]]; then - mkdir output - cp "target/${{ matrix.target }}/release/upgrid" output/upgrid - fi + mkdir output + cp "target/${{ matrix.target }}/release/upgrid" output/upgrid tar --create --gzip --file "$archive" --directory output upgrid - name: Package Windows binary diff --git a/README.md b/README.md index 207f02f..af340c9 100644 --- a/README.md +++ b/README.md @@ -21,14 +21,14 @@ Every Node exposes the same authenticated API and responsive WebUI. Followers tr ## Quick start -Docker is the preferred installation method. Every `main` commit is published as a multi-architecture image named `main-<6-character-commit>`, while stable releases use their Git tag. Start a single Node with persistent storage using `v0.1.0`: +Docker is the preferred installation method. Stable releases use their Git tag and update `latest`. Every `main` commit uses `main-<6-character-commit>` and updates `latest-unstable`. Start a single Node with persistent storage using the latest stable release: ```sh docker run --name upgrid \ --publish 8080:8080 \ --publish 11451:11451/udp \ --volume upgrid-data:/var/lib/upgrid \ - ghcr.io/george-miao/upgrid:v0.1.0 + ghcr.io/george-miao/upgrid:latest ``` Open [http://127.0.0.1:8080/setup](http://127.0.0.1:8080/setup), review the generated Node name, and choose **Create new Cluster**. Setup creates the first replicated Operator Identity before it can add a notification Channel or service Target. diff --git a/flake.nix b/flake.nix index 70bb17e..078837e 100644 --- a/flake.nix +++ b/flake.nix @@ -1,5 +1,5 @@ { - description = "UpGrid dev shell"; + description = "UpGrid"; inputs = { nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable"; @@ -20,25 +20,90 @@ pkgs = import nixpkgs { inherit system overlays; }; - in - with pkgs; { - devShells.default = mkShell { - buildInputs = [ - cargo-feature - curl - jq - nodejs_22 - openssl - pkg-config - pnpm - ripgrep - lldb_21 - (rust-bin.selectLatestNightlyWith (toolchain: - toolchain.default.override { - extensions = ["rust-src"]; - })) + inherit (pkgs) lib; + rustToolchain = pkgs.rust-bin.selectLatestNightlyWith (toolchain: toolchain.minimal); + rustPlatform = pkgs.makeRustPlatform { + cargo = rustToolchain; + rustc = rustToolchain; + }; + source = pkgs.nix-gitignore.gitignoreSourcePure [./.gitignore] ./.; + upgrid = rustPlatform.buildRustPackage { + pname = "upgrid"; + version = "0.1.0"; + src = source; + cargoBuildFlags = ["-p" "upgrid"]; + buildAndTestSubdir = "."; + cargoLock = { + lockFile = ./Cargo.lock; + outputHashes = { + "openraft-0.10.0-alpha.33" = "sha256-IhUCyRg+MvR4UBvD1UphmzSdG7HBhcyHJw9L1eNuyKI="; + "tarpc-0.36.0" = "sha256-RV0LUj0+5DZ2Qa1JsW+BSkA+mAJIvgWJEuPoNelfJ5I="; + }; + }; + doCheck = false; + meta.mainProgram = "upgrid"; + }; + runtimeRoot = pkgs.runCommand "upgrid-runtime-root" {} '' + install -D -m 0755 ${upgrid}/bin/upgrid $out/usr/local/bin/upgrid + ''; + container = pkgs.dockerTools.buildLayeredImage { + name = "upgrid"; + tag = "nix"; + contents = [ + runtimeRoot + pkgs.dockerTools.caCertificates + ]; + extraCommands = '' + mkdir -p etc var/lib/upgrid + printf 'upgrid:x:10001:10001:UpGrid:/var/lib/upgrid:/sbin/nologin\n' > etc/passwd + printf 'upgrid:x:10001:\n' > etc/group + ''; + fakeRootCommands = '' + chown 10001:10001 var/lib/upgrid + ''; + config = { + Entrypoint = ["/usr/local/bin/upgrid"]; + Env = [ + "UPGRID_BIND=0.0.0.0:8080" + "UPGRID_DATA_DIR=/var/lib/upgrid" ]; + ExposedPorts = { + "8080/tcp" = {}; + "11451/udp" = {}; + }; + User = "10001:10001"; + Volumes = { + "/var/lib/upgrid" = {}; + }; + }; + }; + in { + devShells.default = pkgs.mkShell { + buildInputs = [ + pkgs.cargo-feature + pkgs.curl + pkgs.jq + pkgs.nodejs_22 + pkgs.openssl + pkgs.pkg-config + pkgs.pnpm + pkgs.ripgrep + pkgs.lldb_21 + (pkgs.rust-bin.selectLatestNightlyWith (toolchain: + toolchain.default.override { + extensions = ["rust-src"]; + })) + ]; + }; + + packages = + { + default = upgrid; + inherit upgrid; + } + // lib.optionalAttrs pkgs.stdenv.hostPlatform.isLinux { + inherit container; }; - } + } ); } diff --git a/website/src/content/docs/getting-started/installation.md b/website/src/content/docs/getting-started/installation.md index 182fd45..ca5dd1b 100644 --- a/website/src/content/docs/getting-started/installation.md +++ b/website/src/content/docs/getting-started/installation.md @@ -3,7 +3,7 @@ title: Install UpGrid description: Install the published container or a precompiled Linux binary. --- -Docker is the preferred way to install UpGrid. Tagged releases also provide precompiled Linux binaries for hosts where a container runtime is not appropriate. Build from source only for development or an unsupported target. +Docker is the preferred way to install UpGrid. Tagged releases also provide precompiled Linux binaries for hosts where a container runtime is not appropriate. Build from source only for development or an unsupported target. Use `latest` for the current release or `latest-unstable` for the newest image from `main`. ## Run with Docker @@ -12,7 +12,7 @@ docker run --name upgrid \ --publish 8080:8080 \ --publish 11451:11451/udp \ --volume upgrid-data:/var/lib/upgrid \ - ghcr.io/george-miao/upgrid:v0.1.0 + ghcr.io/george-miao/upgrid:latest ``` The image listens for HTTP on port `8080`, exposes QUIC/Raft on UDP port `11451`, and stores durable state in `/var/lib/upgrid`. Open `/setup` to create the Cluster's first Operator Identity. Set `UPGRID_RAFT_URL` to an advertised hostname reachable by every Node before building a multi-Node Cluster.