Merge pull request #216 from kerberos-io/feature/upgrade-build-process-avoid-base

feature/upgrade-build-process-avoid-base

.github/workflows/docker.yml

@@ -14,7 +14,7 @@ env:
 
 jobs:
   build-amd64:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     permissions:
       contents: write
     strategy:
@@ -32,53 +32,36 @@ jobs:
         id: short-sha
         with:
           length: 7
-      - name: Set up QEMU
+      - name: Run Build
-        uses: docker/setup-qemu-action@v2
+        run: |
-      - name: Set up Docker Buildx
+          docker build -t ${{matrix.architecture}} .
-        uses: docker/setup-buildx-action@v2
+          CID=$(docker create ${{matrix.architecture}})
-      - name: Available platforms
+          docker cp ${CID}:/home/agent ./output-${{matrix.architecture}}
-        run: echo ${{ steps.buildx.outputs.platforms }}
+          docker rm ${CID}
-      - name: Run Buildx
-        run: docker buildx build --platform linux/$(echo ${{matrix.architecture}} | tr - /) -t $REPO-arch:arch-${{matrix.architecture}}-${{github.event.inputs.tag || github.ref_name}} --push .
-      - name: Create new and append to manifest
-        run: docker buildx imagetools create -t $REPO:${{ github.event.inputs.tag || github.ref_name }} $REPO-arch:arch-${{matrix.architecture}}-${{github.event.inputs.tag || github.ref_name}}
-      - name: Create new and append to manifest latest
-        run: docker buildx imagetools create -t $REPO:latest $REPO-arch:arch-${{matrix.architecture}}-${{github.event.inputs.tag || github.ref_name}}
-        if: github.event.inputs.tag == 'test'
-      - name: Run Buildx with output
-        run: docker buildx build --platform linux/$(echo ${{matrix.architecture}} | tr - /) -t $REPO-arch:arch-$(echo ${{matrix.architecture}} | tr / -)-${{github.event.inputs.tag || github.ref_name}} --output type=tar,dest=output-${{matrix.architecture}}.tar .
       - name: Strip binary
-        run: mkdir -p output/ && tar -xf output-${{matrix.architecture}}.tar -C output && rm output-${{matrix.architecture}}.tar && cd output/ && tar -cf ../agent-${{matrix.architecture}}.tar -C home/agent . && rm -rf output
+        run: tar -cf agent-${{matrix.architecture}}.tar -C output-${{matrix.architecture}} . && rm -rf output-${{matrix.architecture}}
-      - name: Create a release
+      - name: Build and push Docker image
-        uses: ncipollo/release-action@v1
+        run: |
+          docker tag ${{matrix.architecture}} $REPO-arch:arch-${{matrix.architecture}}-${{github.event.inputs.tag || github.ref_name}}
+          docker push $REPO-arch:arch-${{matrix.architecture}}-${{github.event.inputs.tag || github.ref_name}}
+      - name: Create new manifest
+        run: docker manifest create $REPO:${{ github.event.inputs.tag || github.ref_name }} $REPO-arch:arch-${{matrix.architecture}}-${{github.event.inputs.tag || github.ref_name}}
+      - name: Create latest manifest
+        run: docker manifest create $REPO:latest $REPO-arch:arch-${{matrix.architecture}}-${{github.event.inputs.tag || github.ref_name}}
+        if: github.event.inputs.tag == 'test'
+      - name: Upload artifact
+        uses: actions/upload-artifact@v4
         with:
-          latest: true
+          name: agent-${{matrix.architecture}}.tar
-          allowUpdates: true
+          path: agent-${{matrix.architecture}}.tar
-          name: ${{ github.event.inputs.tag || github.ref_name }}
+          
-          tag: ${{ github.event.inputs.tag || github.ref_name }}
+  build-arm64:
-          generateReleaseNotes: false
+    runs-on: ubuntu-24.04-arm 
-          omitBodyDuringUpdate: true
-          artifacts: "agent-${{matrix.architecture}}.tar"
-      # Taken from GoReleaser's own release workflow.
-      # The available Snapcraft Action has some bugs described in the issue below.
-      # The mkdirs are a hack for https://github.com/goreleaser/goreleaser/issues/1715.
-      #- name: Setup Snapcraft
-      #  run: |
-      #    sudo apt-get update
-      #    sudo apt-get -yq --no-install-suggests --no-install-recommends install snapcraft
-      #    mkdir -p $HOME/.cache/snapcraft/download
-      #    mkdir -p $HOME/.cache/snapcraft/stage-packages
-      #- name: Use Snapcraft
-      #  run: tar -xf agent-${{matrix.architecture}}.tar && snapcraft
-  build-other:
-    runs-on: ubuntu-latest
     permissions:
       contents: write
-    needs: build-amd64
     strategy:
       matrix:
-        architecture: [arm64, arm-v7, arm-v6]
+        architecture: [arm64]
-        #architecture: [arm64, arm-v7]
     steps:
       - name: Login to DockerHub
         uses: docker/login-action@v2
@@ -91,23 +74,55 @@ jobs:
         id: short-sha
         with:
           length: 7
-      - name: Set up QEMU
+      - name: Run Build
-        uses: docker/setup-qemu-action@v2
+        run: |
-      - name: Set up Docker Buildx
+          docker build -t ${{matrix.architecture}} -f Dockerfile.arm64 .
-        uses: docker/setup-buildx-action@v2
+          CID=$(docker create ${{matrix.architecture}})
-      - name: Available platforms
+          docker cp ${CID}:/home/agent ./output-${{matrix.architecture}}
-        run: echo ${{ steps.buildx.outputs.platforms }}
+          docker rm ${CID}
-      - name: Run Buildx
-        run: docker buildx build --platform linux/$(echo ${{matrix.architecture}} | tr - /) -t $REPO-arch:arch-${{matrix.architecture}}-${{github.event.inputs.tag || github.ref_name}} --push .
-      - name: Create new and append to manifest
-        run: docker buildx imagetools create --append -t $REPO:${{ github.event.inputs.tag || github.ref_name }} $REPO-arch:arch-${{matrix.architecture}}-${{github.event.inputs.tag || github.ref_name}}
-      - name: Create new and append to manifest latest
-        run: docker buildx imagetools create --append -t $REPO:latest $REPO-arch:arch-${{matrix.architecture}}-${{github.event.inputs.tag || github.ref_name}}
-        if: github.event.inputs.tag == 'test'
-      - name: Run Buildx with output
-        run: docker buildx build --platform linux/$(echo ${{matrix.architecture}} | tr - /) -t $REPO-arch:arch-$(echo ${{matrix.architecture}} | tr / -)-${{github.event.inputs.tag || github.ref_name}} --output type=tar,dest=output-${{matrix.architecture}}.tar .
       - name: Strip binary
-        run: mkdir -p output/ && tar -xf output-${{matrix.architecture}}.tar -C output && rm output-${{matrix.architecture}}.tar && cd output/ && tar -cf ../agent-${{matrix.architecture}}.tar -C home/agent . && rm -rf output
+        run: tar -cf agent-${{matrix.architecture}}.tar -C output-${{matrix.architecture}} . && rm -rf output-${{matrix.architecture}}
+      - name: Build and push Docker image
+        run: |
+          docker tag ${{matrix.architecture}} $REPO-arch:arch-${{matrix.architecture}}-${{github.event.inputs.tag || github.ref_name}}
+          docker push $REPO-arch:arch-${{matrix.architecture}}-${{github.event.inputs.tag || github.ref_name}}
+      - name: Upload artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: agent-${{matrix.architecture}}.tar
+          path: agent-${{matrix.architecture}}.tar
+
+  create-manifest:
+    runs-on: ubuntu-24.04
+    needs: [build-amd64, build-arm64]
+    steps:
+      - name: Login to DockerHub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+      - name: Create and push multi-arch manifest
+        run: |
+          docker manifest create $REPO:${{ github.event.inputs.tag || github.ref_name }} \
+            $REPO-arch:arch-amd64-${{github.event.inputs.tag || github.ref_name}} \
+            $REPO-arch:arch-arm64-${{github.event.inputs.tag || github.ref_name}}
+          docker manifest push $REPO:${{ github.event.inputs.tag || github.ref_name }}
+      - name: Create and push latest manifest
+        run: |
+          docker manifest create $REPO:latest \
+            $REPO-arch:arch-amd64-${{github.event.inputs.tag || github.ref_name}} \
+            $REPO-arch:arch-arm64-${{github.event.inputs.tag || github.ref_name}}
+          docker manifest push $REPO:latest
+        if: github.event.inputs.tag == 'test'
+
+  create-release:
+    runs-on: ubuntu-24.04
+    needs: [build-amd64, build-arm64]
+    permissions:
+      contents: write
+    steps:
+      - name: Download all artifacts
+        uses: actions/download-artifact@v4
       - name: Create a release
         uses: ncipollo/release-action@v1
         with:
@@ -117,4 +132,4 @@ jobs:
           tag: ${{ github.event.inputs.tag || github.ref_name }}
           generateReleaseNotes: false
           omitBodyDuringUpdate: true
-          artifacts: "agent-${{matrix.architecture}}.tar"
+          artifacts: "agent-*.tar/agent-*.tar"

.github/workflows/pr-build.yml

@@ -0,0 +1,75 @@
+name: Build pull request
+on:
+  pull_request:
+    types: [opened, synchronize]
+
+env:
+  REPO: kerberos/agent
+
+jobs:
+  build-amd64:
+    runs-on: ubuntu-24.04
+    permissions:
+      contents: write
+    strategy:
+      matrix:
+        architecture: [amd64]
+    steps:
+      - name: Login to DockerHub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+      - name: Checkout
+        uses: actions/checkout@v3
+      - uses: benjlevesque/short-sha@v2.1
+        id: short-sha
+        with:
+          length: 7
+      - name: Run Build
+        run: |
+          docker build -t ${{matrix.architecture}} .
+          CID=$(docker create ${{matrix.architecture}})
+          docker cp ${CID}:/home/agent ./output-${{matrix.architecture}}
+          docker rm ${CID}
+      - name: Strip binary
+        run: tar -cf agent-${{matrix.architecture}}.tar -C output-${{matrix.architecture}} . && rm -rf output-${{matrix.architecture}}
+      - name: Upload artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: agent-${{matrix.architecture}}.tar
+          path: agent-${{matrix.architecture}}.tar
+          
+  build-arm64:
+    runs-on: ubuntu-24.04-arm 
+    permissions:
+      contents: write
+    strategy:
+      matrix:
+        architecture: [arm64]
+    steps:
+      - name: Login to DockerHub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+      - name: Checkout
+        uses: actions/checkout@v3
+      - uses: benjlevesque/short-sha@v2.1
+        id: short-sha
+        with:
+          length: 7
+      - name: Run Build
+        run: |
+          docker build -t ${{matrix.architecture}} -f Dockerfile.arm64 .
+          CID=$(docker create ${{matrix.architecture}})
+          docker cp ${CID}:/home/agent ./output-${{matrix.architecture}}
+          docker rm ${CID}
+      - name: Strip binary
+        run: tar -cf agent-${{matrix.architecture}}.tar -C output-${{matrix.architecture}} . && rm -rf output-${{matrix.architecture}}
+      - name: Upload artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: agent-${{matrix.architecture}}.tar
+          path: agent-${{matrix.architecture}}.tar
+

.gitignore

@@ -1,6 +1,8 @@
 ui/node_modules
 ui/build
 ui/public/assets/env.js
+.DS_Store
+__debug*
 .idea
 machinery/www
 yarn.lock

Dockerfile

@@ -1,7 +1,7 @@
 
-ARG BASE_IMAGE_VERSION=70ec57e
+ARG BASE_IMAGE_VERSION=amd64-ddbe40e
 FROM kerberos/base:${BASE_IMAGE_VERSION} AS build-machinery
-LABEL AUTHOR=Kerberos.io
+LABEL AUTHOR=uug.ai
 
 ENV GOROOT=/usr/local/go
 ENV GOPATH=/go

Dockerfile.arm64

@@ -0,0 +1,138 @@
+
+ARG BASE_IMAGE_VERSION=arm64-ddbe40e
+FROM kerberos/base:${BASE_IMAGE_VERSION} AS build-machinery
+LABEL AUTHOR=uug.ai
+
+ENV GOROOT=/usr/local/go
+ENV GOPATH=/go
+ENV PATH=$GOPATH/bin:$GOROOT/bin:/usr/local/lib:$PATH
+ENV GOSUMDB=off
+
+##########################################
+# Installing some additional dependencies.
+
+RUN apt-get upgrade -y && apt-get update && apt-get install -y --fix-missing --no-install-recommends \
+	git build-essential cmake pkg-config unzip libgtk2.0-dev \
+	curl ca-certificates libcurl4-openssl-dev libssl-dev libjpeg62-turbo-dev && \
+	rm -rf /var/lib/apt/lists/*
+
+##############################################################################
+# Copy all the relevant source code in the Docker image, so we can build this.
+
+RUN mkdir -p /go/src/github.com/kerberos-io/agent
+COPY machinery /go/src/github.com/kerberos-io/agent/machinery
+RUN rm -rf /go/src/github.com/kerberos-io/agent/machinery/.env
+
+##################################################################
+# Get the latest commit hash, so we know which version we're running
+COPY .git /go/src/github.com/kerberos-io/agent/.git
+RUN cd /go/src/github.com/kerberos-io/agent/.git && git log --format="%H" -n 1 | head -c7 > /go/src/github.com/kerberos-io/agent/machinery/version
+RUN cat /go/src/github.com/kerberos-io/agent/machinery/version
+
+##################
+# Build Machinery
+
+RUN cd /go/src/github.com/kerberos-io/agent/machinery && \
+	go mod download && \
+	go build -tags timetzdata,netgo,osusergo --ldflags '-s -w -extldflags "-static -latomic"' main.go && \
+	mkdir -p /agent && \
+	mv main /agent && \
+	mv version /agent && \
+	mv data /agent && \
+	mkdir -p /agent/data/cloud && \
+	mkdir -p /agent/data/snapshots && \
+	mkdir -p /agent/data/log && \
+	mkdir -p /agent/data/recordings && \
+	mkdir -p /agent/data/capture-test && \
+	mkdir -p /agent/data/config
+
+####################################
+# Let's create a /dist folder containing just the files necessary for runtime.
+# Later, it will be copied as the / (root) of the output image.
+
+WORKDIR /dist
+RUN cp -r /agent ./
+
+####################################################################################
+# This will collect dependent libraries so they're later copied to the final image.
+
+RUN /dist/agent/main version
+
+FROM node:18.14.0-alpine3.16 AS build-ui
+
+RUN apk update && apk upgrade --available && sync
+
+########################
+# Build Web (React app)
+
+RUN mkdir -p /go/src/github.com/kerberos-io/agent/machinery/www
+COPY ui /go/src/github.com/kerberos-io/agent/ui
+RUN cd /go/src/github.com/kerberos-io/agent/ui && rm -rf yarn.lock && yarn config set network-timeout 300000 && \
+	yarn && yarn build
+
+####################################
+# Let's create a /dist folder containing just the files necessary for runtime.
+# Later, it will be copied as the / (root) of the output image.
+
+WORKDIR /dist
+RUN mkdir -p ./agent && cp -r /go/src/github.com/kerberos-io/agent/machinery/www ./agent/
+
+############################################
+# Publish main binary to GitHub release
+
+FROM alpine:latest
+
+############################
+# Protect by non-root user.
+
+RUN addgroup -S kerberosio && adduser -S agent -G kerberosio && addgroup agent video
+
+#################################
+# Copy files from previous images
+
+COPY --chown=0:0 --from=build-machinery /dist /
+COPY --chown=0:0 --from=build-ui /dist /
+
+RUN apk update && apk add ca-certificates curl libstdc++ libc6-compat --no-cache && rm -rf /var/cache/apk/*
+
+##################
+# Try running agent
+
+RUN mv /agent/* /home/agent/
+RUN /home/agent/main version
+
+#######################
+# Make template config
+
+RUN cp /home/agent/data/config/config.json /home/agent/data/config.template.json
+
+###########################
+# Set permissions correctly
+
+RUN chown -R agent:kerberosio /home/agent/data
+RUN chown -R agent:kerberosio /home/agent/www
+
+###########################
+# Grant the necessary root capabilities to the process trying to bind to the privileged port
+RUN apk add libcap && setcap 'cap_net_bind_service=+ep' /home/agent/main
+
+###################
+# Run non-root user
+
+USER agent
+
+######################################
+# By default the app runs on port 80
+
+EXPOSE 80
+
+######################################
+# Check if agent is still running
+
+HEALTHCHECK CMD curl --fail http://localhost:80 || exit 1   
+
+###################################################
+# Leeeeettttt'ssss goooooo!!!
+# Run the shizzle from the right working directory.
+WORKDIR /home/agent
+CMD ["./main", "-action", "run", "-port", "80"]