Merge pull request #310 from lmnr-ai/dev

Machine manager, upload PDF, api for queue, small fixes, dataset pagination

Author: dinmukhamedm

.github/workflows/backend-build.yml

@@ -0,0 +1,38 @@
+name: Backend build
+
+on:
+  pull_request:
+    types:
+      - synchronize
+      - opened
+      - reopened
+    paths:
+      - 'app-server/**'
+
+jobs:
+  build-push:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+             
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      
+      - name: Cache Docker layers
+        uses: actions/cache@v4
+        with:
+          path: /tmp/.buildx-cache
+          key: ${{ runner.os }}-buildx-app-server-pr-${{ hashFiles('Cargo.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-buildx-app-server-pr-
+
+      - name: Build Docker image
+        uses: docker/build-push-action@v6
+        with:
+          context: ./app-server
+          push: false
+          platforms: linux/amd64
+          cache-from: type=local,src=/tmp/.buildx-cache
+          cache-to: type=local,dest=/tmp/.buildx-cache,mode=max

app-server/Cargo.lock

@@ -66,6 +66,9 @@ sha3 = "0.10.8"
 aws-sdk-s3 = "1.63.0"
 base64 = "0.22.1"
 sodiumoxide = "0.2.7"
+actix-ws = "0.3.0"
+tokio-tungstenite = "*"
+
 
 [build-dependencies]
 tonic-build = "0.12.3"

app-server/Cargo.toml

@@ -1,30 +1,50 @@
-# Uses the bullseye-slim debian image per the rust recommendation.
-FROM rust:1.81-slim-bullseye AS builder
-
-# Install g++ and other build essentials for compiling openssl/tls dependencies
-RUN apt update
-RUN apt install -y build-essential
+# Build stage with cargo chef for dependency caching
+FROM rust:1.81-slim-bullseye AS chef
+WORKDIR /app-server
 
-# Install other openssl / native tls dependencies
-RUN apt-get update
-RUN apt-get install -y \
+# Install build dependencies and cargo-chef
+RUN apt-get update && apt-get install -y \
+    build-essential \
     pkg-config \
     libssl-dev \
     protobuf-compiler \
     libfontconfig1-dev \
-    libfontconfig \ 
-    libclang-dev
+    libfontconfig \
+    libclang-dev \
+    && rm -rf /var/lib/apt/lists/* \
+    && cargo install cargo-chef
+
+# Prepare recipe for dependency caching
+FROM chef AS planner
+COPY . .
+RUN cargo chef prepare --recipe-path recipe.json
 
-# Clean up some unnecessary apt artifacts
-RUN rm -rf /var/lib/apt/lists/*
+# Build dependencies - this layer is cached
+FROM chef AS builder
+COPY --from=planner /app-server/recipe.json recipe.json
+# Build dependencies
+RUN cargo chef cook --release --recipe-path recipe.json
 
-WORKDIR /app-server
+# Build application
 COPY . .
+RUN cargo build --release --all
+
+# Final runtime stage
+FROM debian:bullseye-slim AS runtime
+WORKDIR /app-server
+
+# Install only runtime dependencies
+RUN apt-get update && apt-get install -y \
+    libssl1.1 \
+    libfontconfig1 \
+    ca-certificates \
+    && rm -rf /var/lib/apt/lists/*
+
+COPY --from=builder /app-server/target/release/app-server .
+# Copy data files for name generation
+COPY data/ /app-server/data/
 
 EXPOSE 8000
-ARG DATABASE_URL
-ENV DATABASE_URL=${DATABASE_URL}
-ENV SQLX_OFFLINE=true
-RUN cargo build --release --all
+EXPOSE 8001
 
-CMD ["./target/release/app-server"]
+CMD ["./app-server"]

app-server/Dockerfile

@@ -17,6 +17,15 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
         .out_dir("./src/code_executor/")
         .compile_protos(&[proto_file], &["proto"])?;
 
+    let proto_file = "./proto/machine_manager_grpc.proto";
+
+    tonic_build::configure()
+        .protoc_arg("--experimental_allow_proto3_optional") // for older systems
+        .build_client(true)
+        .build_server(false)
+        .out_dir("./src/machine_manager/")
+        .compile_protos(&[proto_file], &["proto"])?;
+
     tonic_build::configure()
         .protoc_arg("--experimental_allow_proto3_optional") // for older systems
         .build_client(false)

app-server/build.rs

@@ -0,0 +1,54 @@
+syntax = "proto3";
+package machine_manager_service_grpc;
+
+message StartMachineRequest {}
+
+message StartMachineResponse {
+    string machine_id = 1;
+}
+
+enum ComputerAction {
+  KEY = 0;
+  TYPE = 1;
+  MOUSE_MOVE = 2;
+  LEFT_CLICK = 3;
+  LEFT_CLICK_DRAG = 4;
+  RIGHT_CLICK = 5;
+  MIDDLE_CLICK = 6;
+  DOUBLE_CLICK = 7;
+  SCREENSHOT = 8;
+  CURSOR_POSITION = 9;
+}
+
+message ComputerActionCoordinate { 
+  int32 x = 1;
+  int32 y = 2;
+}
+
+message ComputerActionRequest {
+  string machine_id = 1;
+  ComputerAction action = 2;
+  optional string text = 3;
+  optional ComputerActionCoordinate coordinates = 4;
+}
+
+message ComputerActionResponse {
+  optional string output = 1;
+  optional string error = 2;
+  optional string base64_image = 3;
+  optional string system = 4;
+}
+
+message TerminateMachineRequest {
+    string machine_id = 1;
+}
+
+message TerminateMachineResponse {
+    bool success = 1;
+}
+
+service MachineManagerService {
+    rpc StartMachine(StartMachineRequest) returns (StartMachineResponse);
+    rpc TerminateMachine(TerminateMachineRequest) returns (TerminateMachineResponse);
+    rpc ExecuteComputerAction(ComputerActionRequest) returns (ComputerActionResponse);
+}