Added oracle docker files.

README.md

@@ -36,6 +36,7 @@ Refer [Feature Matrix](doc/feature_matrix.md) for detailed features.
 
 [QuickStart Guide: Lightweight (MySQL)](doc/quickstart.md)\
 [QuickStart Guide: Lightweight (PostgreSQL)](doc/quickstart_postgres.md)\
+[QuickStart Guide: Lightweight (Oracle)(Experimental)](doc/quickstart_oracle.md)\
 [QuickStart Guide: Kafka](doc/quickstart_kafka.md)
 
 ## Blog Articles

doc/quickstart_oracle.md

@@ -0,0 +1,90 @@
+Setup Oracle using the enterprise image. You need an account setup to download the enterprise image from container registry(Oracle)
+https://debezium.io/blog/2022/09/30/debezium-oracle-series-part-1/
+
+Start oracle,clickhouse using docker-compose
+`docker compose -f docker-compose-oracle.yml up`
+
+If you are connecting to a diffferent oracle/clickhouse, update the following fields in `config_oracle.yml` before starting sink connector.
+
+**Oracle**
+```
+database.hostname: "localhost"
+database.port: "1521"
+
+database.dbname: "ORCLCDB"
+database.pdb.name: "ORCLPDB1"
+
+#database.pdb.name: "ORCLPDB1"
+
+database.user: "c##dbzuser"
+
+database.password: "dbz"
+
+database.connection.url: "jdbc:oracle:thin:@//localhost:1521/ORCLCDB"
+
+database.server.name: "dbserver1"
+database.connection.adapter: "logminer"
+```
+
+**ClickHouse**
+in the jdbc.url, change `localhost` if its not running locally
+```
+clickhouse.server.url: "localhost"
+
+clickhouse.server.user: "root"
+
+clickhouse.server.password: "root"
+
+clickhouse.server.port: "8123"
+
+offset.storage.jdbc.url: "jdbc:clickhouse://localhost:8123/altinity_sink_connector"
+schema.history.internal.jdbc.url: "jdbc:clickhouse://localhost:8123/altinity_sink_connector"
+
+```
+
+**NOTE**: Currently DDL is not supported, the default is MySQL ANTLR parser , we need to integrate Debezium Oracle ANTLR parser. So the tables have to be manually created in ClickHouse.
+
+
+Start sink connector JAR.
+
+`java -jar clickhouse-debezium-embedded-0.0.4.jar ../docker/config_oracle.yml `
+
+*Testing*
+
+*Oracle*
+```
+SQL> CREATE TABLE customers (id number(9,0) primary key, name varchar2(50));
+SQL> insert into customers values(1008, 'Anne5');
+INSERT INTO customers VALUES (1001, 'Salles Thomas');
+INSERT INTO customers VALUES (1002, 'George Bailey');
+INSERT INTO customers VALUES (1003, 'Edward Walker');
+INSERT INTO customers VALUES (1004, 'Anne Kretchmar');
+SQL> insert into customers values(1005, 'Anne2');
+SQL> insert into customers values(1005, 'Anne2');
+SQL> insert into customers values(1005, 'Anne2');
+
+```
+
+*ClickHouse*
+```
+
+be4cbfdee689 :) use ORCLPDB1
+
+
+be4cbfdee689 :) select * from CUSTOMERS;
+
+SELECT *
+FROM CUSTOMERS
+
+Query id: 6cf7e2cf-dc1c-485f-b50f-77650b07b76a
+
+┌───ID─┬─NAME───────────┬────────────_version─┬─_sign─┐
+│ 1001 │ Salles Thomas  │ 1753390061439000001 │     0 │
+│ 1002 │ George Bailey  │ 1753390061442000002 │     0 │
+│ 1003 │ Edward Walker  │ 1753390061442000003 │     0 │
+│ 1004 │ Anne Kretchmar │ 1753390061442000004 │     0 │
+│ 1005 │ Anne2          │ 1753390061443000005 │     0 │
+│ 1006 │ Anne2          │ 1753390061443000006 │     0 │
+│ 1007 │ Anne3          │ 1753390061443000007 │     0 │
+└──────┴────────────────┴─────────────────────┴───────┘
+```

sink-connector-lightweight/docker/config_oracle.yml

@@ -0,0 +1,176 @@
+#### Some of the properties are part of Debezium PostgreSQL Connector
+#### https://debezium.io/documentation/reference/stable/connectors/postgresql.html
+# name: Unique name for the connector. Attempting to register again with the same name will fail.
+name: "debezium-embedded-oracle"
+
+auto.create.tables.replicated: "true"
+
+# database.hostname: IP address or hostname of the PostgreSQL database server.
+database.hostname: "localhost"
+
+# database.port: Integer port number of the PostgreSQL database server listening for client connections.
+database.port: "1521"
+
+
+database.dbname: "ORCLCDB"
+
+#"database.dbname": "ORCLCDB",
+database.pdb.name: "ORCLPDB1"
+
+table.include.list: "C##DBZUSER.CUSTOMERS"
+
+#database.pdb.name: "ORCLPDB1"
+
+# database.user: Name of the PostgreSQL database user to be used when connecting to the database.
+#database.user: "C##DBZ2"
+database.user: "c##dbzuser"
+# database.password: Password of the PostgreSQL database user to be used when connecting to the database.
+#database.password: "dbzpass"
+
+database.password: "dbz"
+
+database.connection.url: "jdbc:oracle:thin:@//localhost:1521/ORCLCDB"
+
+# database.server.name: The name of the PostgreSQL database from which events are to be captured when not using snapshot mode.
+#database.server.name: "ER54"
+
+database.server.name: "dbserver1"
+database.connection.adapter: "logminer"
+
+# schema.include.list: An optional list of regular expressions that match schema names to be monitored;
+#schema.include.list: public,public2
+
+slot.name: connector2
+
+# table.include.list: An optional list of regular expressions that match fully-qualified table identifiers for tables to be monitored;
+#table.include.list: "public.tm,public.https://kafka.apache.org/documentation/#consumerconfigs_max.poll.interval.mstm2"
+
+# clickhouse.server.url: Specify only the hostname of the Clickhouse Server.
+clickhouse.server.url: "localhost"
+
+# clickhouse.server.user: Clickhouse Server User
+clickhouse.server.user: "root"
+
+# clickhouse.server.password: Clickhouse Server Password
+clickhouse.server.password: "root"
+
+# clickhouse.server.port: Clickhouse Server Port
+clickhouse.server.port: "8123"
+
+# database.allowPublicKeyRetrieval: "true" https://rmoff.net/2019/10/23/debezium-mysql-v8-public-key-retrieval-is-not-allowed/
+database.allowPublicKeyRetrieval: "true"
+
+# snapshot.mode: Debezium can use different modes when it runs a snapshot. The snapshot mode is determined by the snapshot.mode configuration property.
+snapshot.mode: "initial"
+
+# offset.flush.interval.ms: The number of milliseconds to wait before flushing recent offsets to Kafka. This ensures that offsets are committed within the specified time interval.
+offset.flush.interval.ms: 5000
+
+# connector.class: The Java class for the connector. This must be set to io.debezium.connector.postgresql.PostgresConnector.
+connector.class: "io.debezium.connector.oracle.OracleConnector"
+
+# offset.storage: The Java class that implements the offset storage strategy. This must be set to io.debezium.storage.jdbc.offset.JdbcOffsetBackingStore.
+offset.storage: "io.debezium.storage.jdbc.offset.JdbcOffsetBackingStore"
+
+# offset.storage.jdbc.offset.table.name: The name of the database table where connector offsets are to be stored.
+offset.storage.jdbc.table.name: "altinity_sink_connector.replica_source_info"
+
+# offset.storage.jdbc.url: The JDBC URL for the database where connector offsets are to be stored.
+offset.storage.jdbc.url: "jdbc:clickhouse://localhost:8123/altinity_sink_connector"
+
+# offset.storage.jdbc.user: The name of the database user to be used when connecting to the database where connector offsets are to be stored.
+offset.storage.jdbc.user: "root"
+
+# offset.storage.jdbc.password: The password of the database user to be used when connecting to the database where connector offsets are to be stored.
+offset.storage.jdbc.password: "root"
+
+disable.ddl: "true"
+
+# offset.storage.jdbc.offset.table.ddl: The DDL statement used to create the database table where connector offsets are to be stored.
+offset.storage.jdbc.table.ddl: "CREATE TABLE if not exists %s
+(
+    `id` String,
+    `offset_key` String,
+    `offset_val` String,
+    `record_insert_ts` DateTime,
+    `record_insert_seq` UInt64,
+    `_version` UInt64 MATERIALIZED toUnixTimestamp64Nano(now64(9))
+)
+ENGINE = ReplacingMergeTree(_version)
+ORDER BY id
+SETTINGS index_granularity = 8192"
+offset.storage.jdbc.table.delete: "delete from %s where 1=1"
+
+
+# schema.history.internal: The Java class that implements the schema history strategy. This must be set to io.debezium.storage.jdbc.history.JdbcSchemaHistory.
+schema.history.internal: "io.debezium.storage.jdbc.history.JdbcSchemaHistory"
+
+# schema.history.internal.jdbc.url: The JDBC URL for the database where connector schema history is to be stored.
+schema.history.internal.jdbc.url: "jdbc:clickhouse://localhost:8123/altinity_sink_connector"
+
+# schema.history.internal.jdbc.user: The name of the database user to be used when connecting to the database where connector schema history is to be stored.
+schema.history.internal.jdbc.user: "root"
+
+# schema.history.internal.jdbc.password: The password of the database user to be used when connecting to the database where connector schema history is to be stored.
+schema.history.internal.jdbc.password: "root"
+
+# schema.history.internal.jdbc.schema.history.table.ddl: The DDL statement used to create the database table where connector schema history is to be stored.(Advanced)
+schema.history.internal.jdbc.schema.history.table.ddl: "CREATE TABLE if not exists %s
+(`id` VARCHAR(36) NOT NULL, `history_data` VARCHAR(65000), `history_data_seq` INTEGER, `record_insert_ts` TIMESTAMP NOT NULL, `record_insert_seq` INTEGER NOT NULL) ENGINE=ReplacingMergeTree(record_insert_seq) order by id"
+
+# schema.history.internal.jdbc.schema.history.table.name: The name of the database table where connector schema history is to be stored.
+schema.history.internal.jdbc.schema.history.table.name: "altinity_sink_connector.replicate_schema_history"
+
+
+# enable.snapshot.ddl: If set to true, the connector wil parse the DDL statements as part of initial load.
+enable.snapshot.ddl: "true"
+
+# auto.create.tables: If set to true, the connector will create the database tables for the destination tables if they do not already exist.
+auto.create.tables: "true"
+
+# database.dbname: The name of the PostgreSQL database from which events are to be captured when not using snapshot mode.
+#database.dbname: "public"
+
+clickhouse.jdbc.params: "keepalive.timeout=3,max_buffer_size=1000000,socket_timeout=30000,connection_timeout=30000"
+
+# clickhouse.datetime.timezone: This timezone will override the default timezone of ClickHouse server. Timezone columns will be set to this timezone.
+#clickhouse.datetime.timezone: "UTC"
+
+# skip_replica_start: If set to true, the connector will skip replication on startup. sink-connector-client start_replica will start replication.
+#skip_replica_start: "false"
+
+# binary.handling.mode: The mode for handling binary values. Possible values are bytes, base64, and decode. The default is bytes.
+#binary.handling.mode: "base64"
+
+# ignore_delete: If set to true, the connector will ignore delete events. The default is false.
+#ignore_delete: "true"
+
+#Metrics (Prometheus target), required for Grafana Dashboard
+metrics.enable: "true"
+
+#disable.ddl: If set to true, the connector will ignore DDL events. The default is false.
+#disable.ddl: "false"
+
+#disable.drop.truncate: If set to true, the connector will ignore drop and truncate events. The default is false.
+#disable.drop.truncate: "false"
+
+replica.status.view: "CREATE VIEW %s.show_replica_status
+(
+    `seconds_behind_source` Int32,
+    `utc_time` DateTime('UTC'),
+    `local_time` DateTime
+) AS
+SELECT
+    now() - fromUnixTimestamp(toInt32(JSONExtractUInt(offset_val, 'ts_usec') / 1000000)) AS seconds_behind_source,
+    toDateTime(fromUnixTimestamp(toInt32(JSONExtractUInt(offset_val, 'ts_usec') / 1000000)), 'UTC') AS utc_time,
+    fromUnixTimestamp(toInt32(JSONExtractUInt(offset_val, 'ts_usec') / 1000000)) AS local_time
+FROM altinity_sink_connector.replica_source_info
+SETTINGS final = 1"
+
+
+# Required for fixing bug with freeze in debezium loop.
+use.nongraceful.disconnect: "true"
+database.keep.alive.interval.`ms: "30000"  #Send keepalive every 30 seconds
+database.connection.reconnect.backoff.ms: "1000"
+database.connection.reconnect.backoff.max.ms: "10000"
+database.ssl.mode: "disabled"

sink-connector-lightweight/docker/docker-compose-oracle.yml

@@ -0,0 +1,61 @@
+version: "3.4"
+
+# Ubuntu , set this for redpanda to start
+# https://sort.veritas.com/public/documents/HSO/2.0/linux/productguides/html/hfo_admin_ubuntu/ch04s03.htm
+
+# Clickhouse Table Schema
+# create table test(id int, message String) ENGINE=MergeTree() PRIMARY KEY id;
+
+services:
+  oracle:
+    extends:
+      file: oracle-service.yml
+      service: oracle
+
+
+  clickhouse:
+    extends:
+      file: clickhouse-service.yml
+      service: clickhouse
+    depends_on:
+      zookeeper:
+        condition: service_healthy
+
+  zookeeper:
+    extends:
+      file: zookeeper-service.yml
+      service: zookeeper
+
+  # clickhouse-sink-connector-lt:
+  #   extends:
+  #     file: clickhouse-sink-connector-lt-service.yml
+  #     service: clickhouse-sink-connector-lt
+  #   depends_on:
+  #     - clickhouse
+  #   extra_hosts:
+  #     - "host.docker.internal:host-gateway"
+  #   environment:
+  #     JAVA_OPTS: >
+  #       -Xmx5G
+  #       -Xms128m
+  #   volumes:
+  #     - ./config_oracle.yml:/config.yml
+
+  ### MONITORING ####
+  prometheus:
+    extends:
+      file: prometheus-service.yml
+      service: prometheus
+
+
+  grafana:
+    extends:
+      file: grafana-service.yml
+      service: grafana
+    volumes:
+      - ./config/grafana/config/dashboard.yml:/etc/grafana/provisioning/dashboards/dashboard.yml
+      - ./config/grafana/config/datasource.yml:/etc/grafana/provisioning/datasources/datasource.yml
+      - ./config/grafana/config/altinity_sink_connector.json:/var/lib/grafana/dashboards/altinity_sink_connector.json
+    depends_on:
+      - prometheus
+    ## END OF MONITORING ###

sink-connector-lightweight/docker/oracle-service.yml

@@ -0,0 +1,19 @@
+version: "3.4"
+
+services:
+  oracle:
+    image: gvenzl/oracle-xe:21.3.0-slim
+    restart: always
+    ports:
+      - "1521:1521"
+    environment:
+      - ORACLE_PWD=dbz
+      - ORACLE_USER=debezium
+      - ORACLE_DATABASE=ORCLCDB
+      - ORACLE_PASSWORD=dbz
+      - ORACLE_PDB=ORCLPDB1
+      - APP_USER=debezium
+      - APP_USER_PASSWORD=dbz
+    volumes:
+      - ../sql/init_oracle.sql:/docker-entrypoint-initdb.d/init_oracle.sql
+    # No need for a command override; the image entrypoint is sufficient