Incremental web model

incremental/sql-runner/playbooks/incremental-page-views-manifest.yml.tmpl

@@ -0,0 +1,18 @@
+:targets:
+  - :name:          # insert name
+    :type:          redshift
+    :host:          # insert host
+    :database:      # insert database name (default: snowplow)
+    :port:          # insert database port (default: 5439)
+    :username:      # insert Redshift user name (default: datamodeling)
+    :password:      # insert Redshift user password
+:variables:
+    :input_schema:  # insert schema name (default: atomic)
+    :output_schema: # insert schema name (default: derived)
+    :start_date:    # insert start date (must be a valid Redshift DATE, eg 2018-01-01)
+:steps:
+  - :name: 00-setup-page-views-manifest
+    :queries:
+      - :name: 00-setup-page-views-manifest
+        :file: incremental-web-model/00-setup/00-setup-page-views-manifest.sql
+        :template: true

incremental/sql-runner/playbooks/incremental-page-views.yml.tmpl

@@ -0,0 +1,95 @@
+:targets:
+  - :name:           # insert name
+    :type:           redshift
+    :host:           # insert host
+    :database:       # insert database name (default: snowplow)
+    :port:           # insert database port (default: 5439)
+    :username:       # insert Redshift user name (default: datamodeling)
+    :password:       # insert Redshift user password
+:variables:
+    :input_schema:   # insert schema name (default: atomic)
+    :scratch_schema: # insert schema name (default: scratch)
+    :output_schema:  # insert schema name (default: derived)
+    :update_cadence: # insert time period (must be a valid Redshift INTERVAL, eg 1 week)
+    :start_date:     # insert start date (must be a valid Redshift DATE, eg 2018-01-01)
+:steps:
+  - :name: 00-setup-page-views
+    :queries:
+      - :name: 00-setup-page-views
+        :file: incremental-web-model/01-page-views/00-setup-page-views.sql
+        :template: true
+  - :name: 01-select-etl-tstamps
+    :queries:
+      - :name: 01-select-etl-tstamps
+        :file: incremental-web-model/01-page-views/01-select-etl-tstamps.sql
+        :template: true
+  - :name: 02-select-event-ids
+    :queries:
+      - :name: 02-select-event-ids
+        :file: incremental-web-model/01-page-views/02-select-event-ids.sql
+        :template: true
+  - :name: 03-select-page-view-ids
+    :queries:
+      - :name: 03-select-page-view-ids
+        :file: incremental-web-model/01-page-views/03-select-page-view-ids.sql
+        :template: true
+  - :name: 04-select-ids
+    :queries:
+      - :name: 04-select-ids
+        :file: incremental-web-model/01-page-views/04-select-ids.sql
+        :template: true
+  - :name: 05-page-view-events
+    :queries:
+      - :name: 05-page-view-events
+        :file: incremental-web-model/01-page-views/05-page-view-events.sql
+        :template: true
+  - :name: 06-page-view-time
+    :queries:
+      - :name: 06-page-view-time
+        :file: incremental-web-model/01-page-views/06-page-view-time.sql
+        :template: true
+  - :name: 07-page-view-rank
+    :queries:
+      - :name: 07-page-view-rank
+        :file: incremental-web-model/01-page-views/07-page-view-rank.sql
+        :template: true
+  - :name: 08-events-scroll-depth
+    :queries:
+      - :name: 08-events-scroll-depth
+        :file: incremental-web-model/01-page-views/08-events-scroll-depth.sql
+        :template: true
+  - :name: 09-page-views
+    :queries:
+      - :name: 09-page-views
+        :file: incremental-web-model/01-page-views/09-page-views.sql
+        :template: true
+  - :name: 10-delete-page-views
+    :queries:
+      - :name: 10-delete-page-views
+        :file: incremental-web-model/01-page-views/10-delete-page-views.sql
+        :template: true
+  - :name: 11-append-page-views
+    :queries:
+      - :name: 11-append-page-views
+        :file: incremental-web-model/01-page-views/11-append-page-views.sql
+        :template: true
+  - :name: 12-append-manifest
+    :queries:
+      - :name: 12-append-manifest
+        :file: incremental-web-model/01-page-views/12-append-manifest.sql
+        :template: true
+  - :name: 13-cleanup
+    :queries:
+      - :name: 13-cleanup
+        :file: incremental-web-model/01-page-views/13-cleanup.sql
+        :template: true
+  - :name: 14-vacuum
+    :queries:
+      - :name: 14-vacuum
+        :file: incremental-web-model/01-page-views/14-vacuum.sql
+        :template: true
+  - :name: 15-analyze
+    :queries:
+      - :name: 15-analyze
+        :file: incremental-web-model/01-page-views/15-analyze.sql
+        :template: true

incremental/sql-runner/playbooks/incremental-sessions-manifest.yml.tmpl

@@ -0,0 +1,18 @@
+:targets:
+  - :name:          # insert name
+    :type:          redshift
+    :host:          # insert host
+    :database:      # insert database name (default: snowplow)
+    :port:          # insert database port (default: 5439)
+    :username:      # insert Redshift user name (default: datamodeling)
+    :password:      # insert Redshift user password
+:variables:
+    :input_schema:  # insert schema name (default: atomic)
+    :output_schema: # insert schema name (default: derived)
+    :start_date:    # insert start date (must be a valid Redshift DATE, eg 2018-01-01)
+:steps:
+  - :name: 01-setup-sessions-manifest
+    :queries:
+      - :name: 01-setup-sessions-manifest
+        :file: incremental-web-model/00-setup/01-setup-sessions-manifest.sql
+        :template: true

incremental/sql-runner/playbooks/incremental-sessions.yml.tmpl

@@ -0,0 +1,59 @@
+:targets:
+  - :name:          # insert name
+    :type:          redshift
+    :host:          # insert host
+    :database:      # insert database name (default: snowplow)
+    :port:          # insert database port (default: 5439)
+    :username:      # insert Redshift user name (default: datamodeling)
+    :password:      # insert Redshift user password
+:variables:
+    :input_schema:   # insert schema name (default: atomic)
+    :scratch_schema: # insert schema name (default: scratch)
+    :output_schema:  # insert schema name (default: derived)
+    :update_cadence: # insert time period (must be a valid Redshift INTERVAL, eg 1 week)
+:steps:
+  - :name: 00-setup-sessions
+    :queries:
+      - :name: 00-setup-sessions
+        :file: incremental-web-model/02-sessions/00-setup-sessions.sql
+        :template: true
+  - :name: 01-select-page-views
+    :queries:
+      - :name: 01-select-page-view-ids
+        :file: incremental-web-model/02-sessions/01-select-page-view-ids.sql
+        :template: true
+  - :name: 02-sessions
+    :queries:
+      - :name: 02-sessions
+        :file: incremental-web-model/02-sessions/02-sessions.sql
+        :template: true
+  - :name: 03-delete-sessions
+    :queries:
+      - :name: 03-delete-sessions
+        :file: incremental-web-model/02-sessions/03-delete-sessions.sql
+        :template: true
+  - :name: 04-append-sessions
+    :queries:
+      - :name: 04-append-sessions
+        :file: incremental-web-model/02-sessions/04-append-sessions.sql
+        :template: true
+  - :name: 05-populate-sessions-manifest
+    :queries:
+      - :name: 05-populate-sessions-manifest
+        :file: incremental-web-model/02-sessions/05-populate-sessions-manifest.sql
+        :template: true
+  - :name: 06-cleanup
+    :queries:
+      - :name: 06-cleanup
+        :file: incremental-web-model/02-sessions/06-cleanup.sql
+        :template: true
+  - :name: 07-vacuum
+    :queries:
+      - :name: 07-vacuum
+        :file: incremental-web-model/02-sessions/07-vacuum.sql
+        :template: true
+  - :name: 08-analyze
+    :queries:
+      - :name: 08-analyze
+        :file: incremental-web-model/02-sessions/08-analyze.sql
+        :template: true

incremental/sql-runner/playbooks/incremental-users-manifest.yml.tmpl

@@ -0,0 +1,18 @@
+:targets:
+  - :name:          # insert name
+    :type:          redshift
+    :host:          # insert host
+    :database:      # insert database name (default: snowplow)
+    :port:          # insert database port (default: 5439)
+    :username:      # insert Redshift user name (default: datamodeling)
+    :password:      # insert Redshift user password
+:variables:
+    :input_schema:  # insert schema name (default: atomic)
+    :output_schema: # insert schema name (default: derived)
+    :start_date:    # insert start date (must be a valid Redshift DATE, eg 2018-01-01)
+:steps:
+  - :name: 02-setup-users-manifest
+    :queries:
+      - :name: 02-setup-users-manifest
+        :file: incremental-web-model/00-setup/02-setup-users-manifest.sql
+        :template: true

incremental/sql-runner/playbooks/incremental-users.yml.tmpl

@@ -0,0 +1,59 @@
+:targets:
+  - :name:          # insert name
+    :type:          redshift
+    :host:          # insert host
+    :database:      # insert database name (default: snowplow)
+    :port:          # insert database port (default: 5439)
+    :username:      # insert Redshift user name (default: datamodeling)
+    :password:      # insert Redshift user password
+:variables:
+    :input_schema:   # insert schema name (default: atomic)
+    :scratch_schema: # insert schema name (default: scratch)
+    :output_schema:  # insert schema name (default: derived)
+    :update_cadence: # insert time period (must be a valid Redshift INTERVAL, eg 1 week)
+:steps:
+  - :name: 00-setup-users
+    :queries:
+      - :name: 00-setup-users
+        :file: incremental-web-model/03-users/00-setup-users.sql
+        :template: true
+  - :name: 01-select-session-ids
+    :queries:
+      - :name: 01-select-session-ids
+        :file: incremental-web-model/03-users/01-select-session-ids.sql
+        :template: true
+  - :name: 02-users
+    :queries:
+      - :name: 02-users
+        :file: incremental-web-model/03-users/02-users.sql
+        :template: true
+  - :name: 03-delete-users
+    :queries:
+      - :name: 03-delete-users
+        :file: incremental-web-model/03-users/03-delete-users.sql
+        :template: true
+  - :name: 04-append-users
+    :queries:
+      - :name: 04-append-users
+        :file: incremental-web-model/03-users/04-append-users.sql
+        :template: true
+  - :name: 05-populate-users-manifest
+    :queries:
+      - :name: 05-populate-users-manifest
+        :file: incremental-web-model/03-users/05-populate-users-manifest.sql
+        :template: true
+  - :name: 06-cleanup
+    :queries:
+      - :name: 06-cleanup
+        :file: incremental-web-model/03-users/06-cleanup.sql
+        :template: true
+  - :name: 07-vacuum
+    :queries:
+      - :name: 07-vacuum
+        :file: incremental-web-model/03-users/07-vacuum.sql
+        :template: true
+  - :name: 08-analyze
+    :queries:
+      - :name: 08-analyze
+        :file: incremental-web-model/03-users/08-analyze.sql
+        :template: true

incremental/sql-runner/sql/incremental-web-model/00-setup/00-setup-page-views-manifest.sql

@@ -0,0 +1,21 @@
+/* Create and seed a manifest table for the page views
+** job.
+*/
+
+-- Create the table if it doesn't exist
+CREATE TABLE IF NOT EXISTS {{.output_schema}}.page_views_manifest (
+  etl_tstamp TIMESTAMP ENCODE ZSTD,
+  CONSTRAINT {{.output_schema}}_page_views_manifest_pk PRIMARY KEY(etl_tstamp)
+)
+DISTSTYLE ALL
+SORTKEY(etl_tstamp);
+
+-- Seed the manifest table
+INSERT INTO {{.output_schema}}.page_views_manifest(etl_tstamp) (
+  SELECT
+    MIN(etl_tstamp)
+  FROM
+    {{.input_schema}}.events
+  WHERE
+    DATE_TRUNC('day', etl_tstamp) = DATE_TRUNC('day', '{{.start_date}}'::TIMESTAMP)
+);

incremental/sql-runner/sql/incremental-web-model/00-setup/01-setup-sessions-manifest.sql

@@ -0,0 +1,21 @@
+/* Create and seed a manifest table for the sessions
+** job.
+*/
+
+-- Create the table if it doesn't exist
+CREATE TABLE IF NOT EXISTS {{.output_schema}}.sessions_manifest (
+  page_view_start_time TIMESTAMP ENCODE ZSTD,
+  CONSTRAINT {{.output_schema}}_sessions_manifest_pk PRIMARY KEY(page_view_start_time)
+)
+DISTKEY(page_view_start_time)
+SORTKEY(page_view_start_time);
+
+-- Seed the manifest table
+INSERT INTO {{.output_schema}}.sessions_manifest (
+  SELECT
+    MIN(page_view_start_time)
+  FROM
+    {{.output_schema}}.page_views
+  WHERE
+    DATE_TRUNC('day', page_view_start_time) = DATE_TRUNC('day', '{{.start_date}}'::TIMESTAMP)
+);

incremental/sql-runner/sql/incremental-web-model/00-setup/02-setup-users-manifest.sql

@@ -0,0 +1,21 @@
+/* Create and seed a manifest table for the users
+** job.
+*/
+
+-- Create the table if it doesn't exist
+CREATE TABLE IF NOT EXISTS {{.output_schema}}.users_manifest (
+  session_start_time TIMESTAMP ENCODE ZSTD,
+  CONSTRAINT {{.output_schema}}_users_manifest_pk PRIMARY KEY(session_start_time)
+)
+DISTKEY(session_start_time)
+SORTKEY(session_start_time);
+
+-- Seed the manifest table
+INSERT INTO {{.output_schema}}.users_manifest (
+  SELECT
+    MIN(session_start_time)
+  FROM
+    {{.output_schema}}.sessions
+  WHERE
+    DATE_TRUNC('day', session_start_time) = DATE_TRUNC('day', '{{.start_date}}'::TIMESTAMP)
+);