feat: add clickhouse to docker files, remove feature flags for clickhouse LAM-405 (#332)

* eval time progression (#210)

* initial work to compare evals

* remove unnecessary div

* design

---------

Co-authored-by: Din <dinmukhamed.mailibay@gmail.com>

* feat: add clickhouse to docker files, remove feature flags for clickhouse LAM-405

* feat: remove brackets

* fix: remove comment

* remove feature flags, FE, app-server

* fix: fixes, and revert back to port 5432

* feat: fix producer, consumer

* adjust postgres port

* fix: remove braces

* fix: add model column, filter

* fix: expand type, cleanup

* fix, remove ports

* feat: updated ports

* fix: remove comments

* update readme

---------

Co-authored-by: Dinmukhamed Mailibay <47117969+dinmukhamedm@users.noreply.github.com>
Co-authored-by: skull8888888 <skull8888888@gmail.com>
Co-authored-by: Din <dinmukhamed.mailibay@gmail.com>
Co-authored-by: Nidhi Singh <120259299+nidhi-wa@users.noreply.github.com>

Author: 4 people

.env

@@ -8,3 +8,4 @@ RABBITMQ_DEFAULT_USER=admin
 RABBITMQ_DEFAULT_PASS=adminpasswd
 
 CLICKHOUSE_USER=default
+POSTGRES_PORT=5433

.gitignore

@@ -1,4 +1,5 @@
 **/.env.local
 **/.env
 .DS_Store
-**/..DS_Store
+**/..DS_Store
+.idea

CONTRIBUTING.md

@@ -23,22 +23,22 @@ We do this to avoid legal issues and disputes, and to stay compliant with releva
 
 Don't get overwhelmed by the number of docker-compose files. Here's a quick overview:
 
-- `docker-compose.yml` is the simplest one that spins up frontend, app-server, and postgres. Good for quickstarts.
+- `docker-compose.yml` is the simplest one that spins up frontend, app-server, clickhouse, and postgres. Good for quickstarts.
 - `docker-compose-full.yml` is the one you want to use for running the full stack locally. This is the best 
 for self-hosting.
 - `docker-compose-local-dev-full.yml` full file for local development. To be used when you make changes
   to the backend. It will only run the dependency services (postgres, qdrant, clickhouse, rabbitmq).
   You will need to run `cargo r`, `pnpm run dev`, and `python server.py` manually.
 - `docker-compose-local-dev.yml` is the one you want to use for local development. It will only
-  run postgres and app-server. Good for frontend changes.
+  run postgres, clickhouse, and app-server. Good for frontend changes.
 - `docker-compose-local-build.yml` will build the services from the source and run them in production mode. This is good for self-hosting with your own changes,
 or for testing the changes after developing on your own and before opening a PR.
 
 | Service | docker-compose.yml | docker-compose-full.yml | docker-compose-local-dev-full.yml | docker-compose-local-dev.yml | docker-compose-local-build.yml |
 |---------|-------------------|------------------------|------------------------------|----------------------------|------------------------------|
 | postgres | ✅ | ✅ | ✅ | ✅ | ✅ |
 | qdrant | ❌ | ✅ | ✅ | ❌ | ✅ |
-| clickhouse | ❌ | ✅ | ✅ | ❌ | ✅ |
+| clickhouse | ✅ | ✅ | ✅ | ✅ | ✅ |
 | rabbitmq | ❌ | ✅ | ✅ | ❌ | ✅ |
 | app-server | ℹ️ | ✅ | 💻 | ℹ️ | 🔧 | 
 | frontend | ℹ️ | ✅ | 💻 | 💻 | 🔧 |

README.md

@@ -48,7 +48,7 @@ cd lmnr
 docker compose up -d
 ```
 
-This will spin up a lightweight version of the stack with Postgres, app-server, and frontend. This is good for a quickstart 
+This will spin up a lightweight version of the stack with Postgres, clickhouse, app-server, and frontend. This is good for a quickstart 
 or for lightweight usage. You can access the UI at http://localhost:5667 in your browser.
 
 For production environment, we recommend using our [managed platform](https://www.lmnr.ai/projects) or `docker compose -f docker-compose-full.yml up -d`. 

app-server/.env.example

@@ -1,6 +1,6 @@
 SEMANTIC_SEARCH_URL=http://localhost:8080
 # postgres://user:password@host:port/dbname
-DATABASE_URL="postgres://postgres:postgres_passwordabc@localhost:5432/postgres"
+DATABASE_URL="postgres://postgres:postgres_passwordabc@localhost:5433/postgres"
 
 PORT=8000
 GRPC_PORT=8001

app-server/src/api/v1/traces.rs

@@ -29,10 +29,12 @@ pub async fn process_traces(
     project_api_key: ProjectApiKey,
     rabbitmq_connection: web::Data<Option<Arc<Connection>>>,
     db: web::Data<DB>,
+    clickhouse: web::Data<clickhouse::Client>,
     cache: web::Data<crate::cache::Cache>,
 ) -> ResponseResult {
     let db = db.into_inner();
     let cache = cache.into_inner();
+    let clickhouse = clickhouse.into_inner().as_ref().clone();
     let request = ExportTraceServiceRequest::decode(body).map_err(|e| {
         anyhow::anyhow!("Failed to decode ExportTraceServiceRequest from bytes. {e}")
     })?;
@@ -57,6 +59,7 @@ pub async fn process_traces(
         project_api_key.project_id,
         rabbitmq_connection,
         db,
+        clickhouse,
         cache,
     )
     .await?;

app-server/src/ch/evaluation_scores.rs

@@ -4,12 +4,8 @@ use clickhouse::Row;
 use serde::{Deserialize, Serialize, Serializer};
 use uuid::Uuid;
 
-use crate::{
-    evaluations::utils::EvaluationDatapointResult,
-    features::{is_feature_enabled, Feature},
-};
-
 use super::utils::chrono_to_nanoseconds;
+use crate::evaluations::utils::EvaluationDatapointResult;
 
 fn serialize_timestamp<S>(timestamp: &DateTime<Utc>, serializer: S) -> Result<S::Ok, S::Error>
 where
@@ -78,10 +74,6 @@ pub async fn insert_evaluation_scores(
         return Ok(());
     }
 
-    if !is_feature_enabled(Feature::FullBuild) {
-        return Ok(());
-    }
-
     let ch_insert = clickhouse.insert("evaluation_scores");
     match ch_insert {
         Ok(mut ch_insert) => {
@@ -284,9 +276,6 @@ pub async fn delete_evaluation_score(
     result_id: Uuid,
     label_id: Uuid,
 ) -> Result<()> {
-    if !is_feature_enabled(Feature::FullBuild) {
-        return Ok(());
-    }
     // Note, this does not immediately physically delete the data.
     // https://clickhouse.com/docs/en/sql-reference/statements/delete
     clickhouse

app-server/src/ch/events.rs

@@ -3,7 +3,7 @@ use clickhouse::Row;
 use serde::Serialize;
 use uuid::Uuid;
 
-use crate::{db::events::Event, features::is_feature_enabled, Feature};
+use crate::db::events::Event;
 
 use super::utils::chrono_to_nanoseconds;
 
@@ -30,9 +30,6 @@ impl CHEvent {
 }
 
 pub async fn insert_events(clickhouse: clickhouse::Client, events: Vec<CHEvent>) -> Result<()> {
-    if !is_feature_enabled(Feature::FullBuild) {
-        return Ok(());
-    }
     if events.is_empty() {
         return Ok(());
     }

app-server/src/ch/labels.rs

@@ -4,10 +4,7 @@ use clickhouse::Row;
 use serde::{Deserialize, Serialize};
 use uuid::Uuid;
 
-use crate::{
-    db::labels::LabelSource,
-    features::{is_feature_enabled, Feature},
-};
+use crate::db::labels::LabelSource;
 
 use super::utils::chrono_to_nanoseconds;
 
@@ -76,10 +73,6 @@ pub async fn insert_label(
     value: f64,
     span_id: Uuid,
 ) -> Result<()> {
-    if !is_feature_enabled(Feature::FullBuild) {
-        return Ok(());
-    }
-
     let label = CHLabel::new(
         project_id,
         class_id,
@@ -120,9 +113,6 @@ pub async fn delete_label(
     span_id: Uuid,
     id: Uuid,
 ) -> Result<()> {
-    if !is_feature_enabled(Feature::FullBuild) {
-        return Ok(());
-    }
     // Note, this does not immediately physically delete the data.
     // https://clickhouse.com/docs/en/sql-reference/statements/delete
     client

app-server/src/ch/spans.rs

@@ -6,7 +6,6 @@ use uuid::Uuid;
 
 use crate::{
     db::spans::{Span, SpanType},
-    features::{is_feature_enabled, Feature},
     traces::spans::SpanUsage,
 };
 
@@ -97,9 +96,6 @@ impl CHSpan {
 }
 
 pub async fn insert_span(clickhouse: clickhouse::Client, span: &CHSpan) -> Result<()> {
-    if !is_feature_enabled(Feature::FullBuild) {
-        return Ok(());
-    }
     let ch_insert = clickhouse.insert("spans");
     match ch_insert {
         Ok(mut ch_insert) => {