Merge pull request #349 from lmnr-ai/dev

Change ports, add clickhouse to lite version

Author: dinmukhamedm

.env

@@ -8,3 +8,4 @@ RABBITMQ_DEFAULT_USER=admin
 RABBITMQ_DEFAULT_PASS=adminpasswd
 
 CLICKHOUSE_USER=default
+POSTGRES_PORT=5433

.gitignore

@@ -1,4 +1,5 @@
 **/.env.local
 **/.env
 .DS_Store
-**/..DS_Store
+**/..DS_Store
+.idea

CONTRIBUTING.md

@@ -23,22 +23,22 @@ We do this to avoid legal issues and disputes, and to stay compliant with releva
 
 Don't get overwhelmed by the number of docker-compose files. Here's a quick overview:
 
-- `docker-compose.yml` is the simplest one that spins up frontend, app-server, and postgres. Good for quickstarts.
+- `docker-compose.yml` is the simplest one that spins up frontend, app-server, clickhouse, and postgres. Good for quickstarts.
 - `docker-compose-full.yml` is the one you want to use for running the full stack locally. This is the best 
 for self-hosting.
 - `docker-compose-local-dev-full.yml` full file for local development. To be used when you make changes
   to the backend. It will only run the dependency services (postgres, qdrant, clickhouse, rabbitmq).
   You will need to run `cargo r`, `pnpm run dev`, and `python server.py` manually.
 - `docker-compose-local-dev.yml` is the one you want to use for local development. It will only
-  run postgres and app-server. Good for frontend changes.
+  run postgres, clickhouse, and app-server. Good for frontend changes.
 - `docker-compose-local-build.yml` will build the services from the source and run them in production mode. This is good for self-hosting with your own changes,
 or for testing the changes after developing on your own and before opening a PR.
 
 | Service | docker-compose.yml | docker-compose-full.yml | docker-compose-local-dev-full.yml | docker-compose-local-dev.yml | docker-compose-local-build.yml |
 |---------|-------------------|------------------------|------------------------------|----------------------------|------------------------------|
 | postgres | ✅ | ✅ | ✅ | ✅ | ✅ |
 | qdrant | ❌ | ✅ | ✅ | ❌ | ✅ |
-| clickhouse | ❌ | ✅ | ✅ | ❌ | ✅ |
+| clickhouse | ✅ | ✅ | ✅ | ✅ | ✅ |
 | rabbitmq | ❌ | ✅ | ✅ | ❌ | ✅ |
 | app-server | ℹ️ | ✅ | 💻 | ℹ️ | 🔧 | 
 | frontend | ℹ️ | ✅ | 💻 | 💻 | 🔧 |

README.md

@@ -48,8 +48,8 @@ cd lmnr
 docker compose up -d
 ```
 
-This will spin up a lightweight version of the stack with Postgres, app-server, and frontend. This is good for a quickstart 
-or for lightweight usage. You can access the UI at http://localhost:3000 in your browser.
+This will spin up a lightweight version of the stack with Postgres, clickhouse, app-server, and frontend. This is good for a quickstart 
+or for lightweight usage. You can access the UI at http://localhost:5667 in your browser.
 
 For production environment, we recommend using our [managed platform](https://www.lmnr.ai/projects) or `docker compose -f docker-compose-full.yml up -d`. 
 

app-server/.env.example

@@ -1,6 +1,6 @@
 SEMANTIC_SEARCH_URL=http://localhost:8080
 # postgres://user:password@host:port/dbname
-DATABASE_URL="postgres://postgres:postgres_passwordabc@localhost:5432/postgres"
+DATABASE_URL="postgres://postgres:postgres_passwordabc@localhost:5433/postgres"
 
 PORT=8000
 GRPC_PORT=8001

app-server/src/api/v1/traces.rs

@@ -29,10 +29,12 @@ pub async fn process_traces(
     project_api_key: ProjectApiKey,
     rabbitmq_connection: web::Data<Option<Arc<Connection>>>,
     db: web::Data<DB>,
+    clickhouse: web::Data<clickhouse::Client>,
     cache: web::Data<crate::cache::Cache>,
 ) -> ResponseResult {
     let db = db.into_inner();
     let cache = cache.into_inner();
+    let clickhouse = clickhouse.into_inner().as_ref().clone();
     let request = ExportTraceServiceRequest::decode(body).map_err(|e| {
         anyhow::anyhow!("Failed to decode ExportTraceServiceRequest from bytes. {e}")
     })?;
@@ -57,6 +59,7 @@ pub async fn process_traces(
         project_api_key.project_id,
         rabbitmq_connection,
         db,
+        clickhouse,
         cache,
     )
     .await?;

app-server/src/ch/evaluation_scores.rs

@@ -4,12 +4,8 @@ use clickhouse::Row;
 use serde::{Deserialize, Serialize, Serializer};
 use uuid::Uuid;
 
-use crate::{
-    evaluations::utils::EvaluationDatapointResult,
-    features::{is_feature_enabled, Feature},
-};
-
 use super::utils::chrono_to_nanoseconds;
+use crate::evaluations::utils::EvaluationDatapointResult;
 
 fn serialize_timestamp<S>(timestamp: &DateTime<Utc>, serializer: S) -> Result<S::Ok, S::Error>
 where
@@ -78,10 +74,6 @@ pub async fn insert_evaluation_scores(
         return Ok(());
     }
 
-    if !is_feature_enabled(Feature::FullBuild) {
-        return Ok(());
-    }
-
     let ch_insert = clickhouse.insert("evaluation_scores");
     match ch_insert {
         Ok(mut ch_insert) => {
@@ -284,9 +276,6 @@ pub async fn delete_evaluation_score(
     result_id: Uuid,
     label_id: Uuid,
 ) -> Result<()> {
-    if !is_feature_enabled(Feature::FullBuild) {
-        return Ok(());
-    }
     // Note, this does not immediately physically delete the data.
     // https://clickhouse.com/docs/en/sql-reference/statements/delete
     clickhouse

app-server/src/ch/events.rs

@@ -3,7 +3,7 @@ use clickhouse::Row;
 use serde::Serialize;
 use uuid::Uuid;
 
-use crate::{db::events::Event, features::is_feature_enabled, Feature};
+use crate::db::events::Event;
 
 use super::utils::chrono_to_nanoseconds;
 
@@ -30,9 +30,6 @@ impl CHEvent {
 }
 
 pub async fn insert_events(clickhouse: clickhouse::Client, events: Vec<CHEvent>) -> Result<()> {
-    if !is_feature_enabled(Feature::FullBuild) {
-        return Ok(());
-    }
     if events.is_empty() {
         return Ok(());
     }

app-server/src/ch/labels.rs

@@ -4,10 +4,7 @@ use clickhouse::Row;
 use serde::{Deserialize, Serialize};
 use uuid::Uuid;
 
-use crate::{
-    db::labels::LabelSource,
-    features::{is_feature_enabled, Feature},
-};
+use crate::db::labels::LabelSource;
 
 use super::utils::chrono_to_nanoseconds;
 
@@ -76,10 +73,6 @@ pub async fn insert_label(
     value: f64,
     span_id: Uuid,
 ) -> Result<()> {
-    if !is_feature_enabled(Feature::FullBuild) {
-        return Ok(());
-    }
-
     let label = CHLabel::new(
         project_id,
         class_id,
@@ -120,9 +113,6 @@ pub async fn delete_label(
     span_id: Uuid,
     id: Uuid,
 ) -> Result<()> {
-    if !is_feature_enabled(Feature::FullBuild) {
-        return Ok(());
-    }
     // Note, this does not immediately physically delete the data.
     // https://clickhouse.com/docs/en/sql-reference/statements/delete
     client

app-server/src/ch/spans.rs

@@ -6,7 +6,6 @@ use uuid::Uuid;
 
 use crate::{
     db::spans::{Span, SpanType},
-    features::{is_feature_enabled, Feature},
     traces::spans::SpanUsage,
 };
 
@@ -97,9 +96,6 @@ impl CHSpan {
 }
 
 pub async fn insert_span(clickhouse: clickhouse::Client, span: &CHSpan) -> Result<()> {
-    if !is_feature_enabled(Feature::FullBuild) {
-        return Ok(());
-    }
     let ch_insert = clickhouse.insert("spans");
     match ch_insert {
         Ok(mut ch_insert) => {