From 92f4fa7b8871f61ee3122f804f700f3cebbb09ec Mon Sep 17 00:00:00 2001 From: salmany Date: Mon, 4 Aug 2025 22:37:59 +0000 Subject: [PATCH 01/11] feat: provide code examples for managedkafka connect clusters and connectors * Managed Kafka Connect Go Code Samples * Adds fake client for testing Connect clusters --- internal/managedkafka/fake/fake.go | 115 ++++++++++- .../connect/clusters/connect_clusters_test.go | 95 +++++++++ .../clusters/create_connect_cluster.go | 74 +++++++ .../clusters/delete_connect_cluster.go | 56 +++++ .../connect/clusters/get_connect_cluster.go | 52 +++++ .../connect/clusters/list_connect_clusters.go | 58 ++++++ .../clusters/update_connect_cluster.go | 76 +++++++ .../connect/connectors/connectors_test.go | 194 ++++++++++++++++++ .../create_bigquery_sink_connector.go | 77 +++++++ .../connectors/create_gcs_sink_connector.go | 78 +++++++ .../create_mirrormaker_connector.go | 85 ++++++++ .../create_pubsub_sink_connector.go | 76 +++++++ .../create_pubsub_source_connector.go | 76 +++++++ .../connect/connectors/delete_connector.go | 53 +++++ .../connect/connectors/get_connector.go | 53 +++++ .../connect/connectors/list_connectors.go | 59 ++++++ .../connect/connectors/pause_connector.go | 53 +++++ .../connect/connectors/restart_connector.go | 53 +++++ .../connect/connectors/resume_connector.go | 53 +++++ .../connect/connectors/stop_connector.go | 53 +++++ .../connect/connectors/update_connector.go | 65 ++++++ managedkafka/go.mod | 80 ++++---- 22 files changed, 1592 insertions(+), 42 deletions(-) create mode 100644 managedkafka/connect/clusters/connect_clusters_test.go create mode 100644 managedkafka/connect/clusters/create_connect_cluster.go create mode 100644 managedkafka/connect/clusters/delete_connect_cluster.go create mode 100644 managedkafka/connect/clusters/get_connect_cluster.go create mode 100644 managedkafka/connect/clusters/list_connect_clusters.go create mode 100644 managedkafka/connect/clusters/update_connect_cluster.go create mode 100644 managedkafka/connect/connectors/connectors_test.go create mode 100644 managedkafka/connect/connectors/create_bigquery_sink_connector.go create mode 100644 managedkafka/connect/connectors/create_gcs_sink_connector.go create mode 100644 managedkafka/connect/connectors/create_mirrormaker_connector.go create mode 100644 managedkafka/connect/connectors/create_pubsub_sink_connector.go create mode 100644 managedkafka/connect/connectors/create_pubsub_source_connector.go create mode 100644 managedkafka/connect/connectors/delete_connector.go create mode 100644 managedkafka/connect/connectors/get_connector.go create mode 100644 managedkafka/connect/connectors/list_connectors.go create mode 100644 managedkafka/connect/connectors/pause_connector.go create mode 100644 managedkafka/connect/connectors/restart_connector.go create mode 100644 managedkafka/connect/connectors/resume_connector.go create mode 100644 managedkafka/connect/connectors/stop_connector.go create mode 100644 managedkafka/connect/connectors/update_connector.go diff --git a/internal/managedkafka/fake/fake.go b/internal/managedkafka/fake/fake.go index f66798971e..0488e39dfa 100644 --- a/internal/managedkafka/fake/fake.go +++ b/internal/managedkafka/fake/fake.go @@ -30,9 +30,11 @@ import ( ) const ( - clusterID = "fake-cluster" - topicID = "fake-topic" - consumerGroupID = "fake-consumergroup" + clusterID = "fake-cluster" + topicID = "fake-topic" + consumerGroupID = "fake-consumergroup" + connectClusterID = "fake-connect-cluster" + connectorID = "fake-connector" ) // The reason why we have a fake server is because testing end-to-end will exceed the deadline of 10 minutes. @@ -41,14 +43,20 @@ type fakeManagedKafkaServer struct { managedkafkapb.UnimplementedManagedKafkaServer } +type fakeManagedKafkaConnectServer struct { + managedkafkapb.UnimplementedManagedKafkaConnectServer +} + func Options(t *testing.T) []option.ClientOption { server := &fakeManagedKafkaServer{} + connectServer := &fakeManagedKafkaConnectServer{} listener, err := net.Listen("tcp", "localhost:0") if err != nil { t.Fatal(err) } gsrv := grpc.NewServer() managedkafkapb.RegisterManagedKafkaServer(gsrv, server) + managedkafkapb.RegisterManagedKafkaConnectServer(gsrv, connectServer) fakeServerAddr := listener.Addr().String() go func() { if err := gsrv.Serve(listener); err != nil { @@ -165,3 +173,104 @@ func (f *fakeManagedKafkaServer) UpdateConsumerGroup(ctx context.Context, req *m Name: consumerGroupID, }, nil } + +// Connect server methods +func (f *fakeManagedKafkaConnectServer) CreateConnectCluster(ctx context.Context, req *managedkafkapb.CreateConnectClusterRequest) (*longrunningpb.Operation, error) { + anypb := &anypb.Any{} + err := anypb.MarshalFrom(req.ConnectCluster) + if err != nil { + return nil, fmt.Errorf("anypb.MarshalFrom got err: %w", err) + } + return &longrunningpb.Operation{ + Done: true, + Result: &longrunningpb.Operation_Response{ + Response: anypb, + }, + }, nil +} + +func (f *fakeManagedKafkaConnectServer) DeleteConnectCluster(ctx context.Context, req *managedkafkapb.DeleteConnectClusterRequest) (*longrunningpb.Operation, error) { + return &longrunningpb.Operation{ + Done: true, + Result: &longrunningpb.Operation_Response{ + Response: &anypb.Any{}, + }, + }, nil +} + +func (f *fakeManagedKafkaConnectServer) GetConnectCluster(ctx context.Context, req *managedkafkapb.GetConnectClusterRequest) (*managedkafkapb.ConnectCluster, error) { + return &managedkafkapb.ConnectCluster{ + Name: connectClusterID, + }, nil +} + +func (f *fakeManagedKafkaConnectServer) ListConnectClusters(ctx context.Context, req *managedkafkapb.ListConnectClustersRequest) (*managedkafkapb.ListConnectClustersResponse, error) { + return &managedkafkapb.ListConnectClustersResponse{ + ConnectClusters: []*managedkafkapb.ConnectCluster{{ + Name: connectClusterID, + }}, + }, nil +} + +func (f *fakeManagedKafkaConnectServer) UpdateConnectCluster(ctx context.Context, req *managedkafkapb.UpdateConnectClusterRequest) (*longrunningpb.Operation, error) { + anypb := &anypb.Any{} + err := anypb.MarshalFrom(req.ConnectCluster) + if err != nil { + return nil, fmt.Errorf("anypb.MarshalFrom got err: %w", err) + } + return &longrunningpb.Operation{ + Done: true, + Result: &longrunningpb.Operation_Response{ + Response: anypb, + }, + }, nil +} + +// Connector methods +func (f *fakeManagedKafkaConnectServer) CreateConnector(ctx context.Context, req *managedkafkapb.CreateConnectorRequest) (*managedkafkapb.Connector, error) { + return req.Connector, nil +} + +func (f *fakeManagedKafkaConnectServer) GetConnector(ctx context.Context, req *managedkafkapb.GetConnectorRequest) (*managedkafkapb.Connector, error) { + return &managedkafkapb.Connector{ + Name: connectorID, + Configs: map[string]string{ + "connector.class": "test.connector", + }, + }, nil +} + +func (f *fakeManagedKafkaConnectServer) ListConnectors(ctx context.Context, req *managedkafkapb.ListConnectorsRequest) (*managedkafkapb.ListConnectorsResponse, error) { + return &managedkafkapb.ListConnectorsResponse{ + Connectors: []*managedkafkapb.Connector{{ + Name: connectorID, + Configs: map[string]string{ + "connector.class": "test.connector", + }, + }}, + }, nil +} + +func (f *fakeManagedKafkaConnectServer) UpdateConnector(ctx context.Context, req *managedkafkapb.UpdateConnectorRequest) (*managedkafkapb.Connector, error) { + return req.Connector, nil +} + +func (f *fakeManagedKafkaConnectServer) DeleteConnector(ctx context.Context, req *managedkafkapb.DeleteConnectorRequest) (*emptypb.Empty, error) { + return &emptypb.Empty{}, nil +} + +func (f *fakeManagedKafkaConnectServer) PauseConnector(ctx context.Context, req *managedkafkapb.PauseConnectorRequest) (*managedkafkapb.PauseConnectorResponse, error) { + return &managedkafkapb.PauseConnectorResponse{}, nil +} + +func (f *fakeManagedKafkaConnectServer) ResumeConnector(ctx context.Context, req *managedkafkapb.ResumeConnectorRequest) (*managedkafkapb.ResumeConnectorResponse, error) { + return &managedkafkapb.ResumeConnectorResponse{}, nil +} + +func (f *fakeManagedKafkaConnectServer) StopConnector(ctx context.Context, req *managedkafkapb.StopConnectorRequest) (*managedkafkapb.StopConnectorResponse, error) { + return &managedkafkapb.StopConnectorResponse{}, nil +} + +func (f *fakeManagedKafkaConnectServer) RestartConnector(ctx context.Context, req *managedkafkapb.RestartConnectorRequest) (*managedkafkapb.RestartConnectorResponse, error) { + return &managedkafkapb.RestartConnectorResponse{}, nil +} diff --git a/managedkafka/connect/clusters/connect_clusters_test.go b/managedkafka/connect/clusters/connect_clusters_test.go new file mode 100644 index 0000000000..422eee7811 --- /dev/null +++ b/managedkafka/connect/clusters/connect_clusters_test.go @@ -0,0 +1,95 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package clusters + +import ( + "bytes" + "fmt" + "strings" + "testing" + "time" + + "github.com/GoogleCloudPlatform/golang-samples/internal/managedkafka/fake" + "github.com/GoogleCloudPlatform/golang-samples/internal/testutil" +) + +const ( + connectClusterPrefix = "connect-cluster" + region = "us-central1" +) + +func TestConnectClusters(t *testing.T) { + tc := testutil.SystemTest(t) + buf := new(bytes.Buffer) + connectClusterID := fmt.Sprintf("%s-%d", connectClusterPrefix, time.Now().UnixNano()) + kafkaClusterID := fmt.Sprintf("kafka-cluster-%d", time.Now().UnixNano()) + options := fake.Options(t) + + // First create a Kafka cluster that the Connect cluster will reference + kafkaClusterPath := fmt.Sprintf("projects/%s/locations/%s/clusters/%s", tc.ProjectID, region, kafkaClusterID) + + t.Run("CreateConnectCluster", func(t *testing.T) { + if err := createConnectCluster(buf, tc.ProjectID, region, connectClusterID, kafkaClusterPath, options...); err != nil { + t.Fatalf("failed to create a connect cluster: %v", err) + } + got := buf.String() + want := "Created connect cluster" + if !strings.Contains(got, want) { + t.Fatalf("createConnectCluster() mismatch got: %s\nwant: %s", got, want) + } + }) + t.Run("GetConnectCluster", func(t *testing.T) { + if err := getConnectCluster(buf, tc.ProjectID, region, connectClusterID, options...); err != nil { + t.Fatalf("failed to get connect cluster: %v", err) + } + got := buf.String() + want := "Got connect cluster" + if !strings.Contains(got, want) { + t.Fatalf("getConnectCluster() mismatch got: %s\nwant: %s", got, want) + } + }) + t.Run("UpdateConnectCluster", func(t *testing.T) { + memoryBytes := int64(13958643712) // 13 GiB in bytes + labels := map[string]string{"environment": "test"} + if err := updateConnectCluster(buf, tc.ProjectID, region, connectClusterID, memoryBytes, labels, options...); err != nil { + t.Fatalf("failed to update connect cluster: %v", err) + } + got := buf.String() + want := "Updated connect cluster" + if !strings.Contains(got, want) { + t.Fatalf("updateConnectCluster() mismatch got: %s\nwant: %s", got, want) + } + }) + t.Run("ListConnectClusters", func(t *testing.T) { + if err := listConnectClusters(buf, tc.ProjectID, region, options...); err != nil { + t.Fatalf("failed to list connect clusters: %v", err) + } + got := buf.String() + want := "Got connect cluster" + if !strings.Contains(got, want) { + t.Fatalf("listConnectClusters() mismatch got: %s\nwant: %s", got, want) + } + }) + t.Run("DeleteConnectCluster", func(t *testing.T) { + if err := deleteConnectCluster(buf, tc.ProjectID, region, connectClusterID, options...); err != nil { + t.Fatalf("failed to delete connect cluster: %v", err) + } + got := buf.String() + want := "Deleted connect cluster" + if !strings.Contains(got, want) { + t.Fatalf("deleteConnectCluster() mismatch got: %s\nwant: %s", got, want) + } + }) +} diff --git a/managedkafka/connect/clusters/create_connect_cluster.go b/managedkafka/connect/clusters/create_connect_cluster.go new file mode 100644 index 0000000000..795096362a --- /dev/null +++ b/managedkafka/connect/clusters/create_connect_cluster.go @@ -0,0 +1,74 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package clusters + +// [START managedkafka_create_connect_cluster] +import ( + "context" + "fmt" + "io" + + "cloud.google.com/go/managedkafka/apiv1/managedkafkapb" + "google.golang.org/api/option" + + managedkafka "cloud.google.com/go/managedkafka/apiv1" +) + +func createConnectCluster(w io.Writer, projectID, region, clusterID, kafkaCluster string, opts ...option.ClientOption) error { + // projectID := "my-project-id" + // region := "us-central1" + // clusterID := "my-connect-cluster" + // kafkaCluster := "projects/my-project-id/locations/us-central1/clusters/my-kafka-cluster" + ctx := context.Background() + client, err := managedkafka.NewManagedKafkaConnectClient(ctx, opts...) + if err != nil { + return fmt.Errorf("managedkafka.NewManagedKafkaConnectClient got err: %w", err) + } + defer client.Close() + + locationPath := fmt.Sprintf("projects/%s/locations/%s", projectID, region) + clusterPath := fmt.Sprintf("%s/connectClusters/%s", locationPath, clusterID) + + // Capacity configuration with 12 vCPU and 12 GiB RAM + capacityConfig := &managedkafkapb.CapacityConfig{ + VcpuCount: 12, + MemoryBytes: 12884901888, // 12 GiB in bytes + } + + connectCluster := &managedkafkapb.ConnectCluster{ + Name: clusterPath, + KafkaCluster: kafkaCluster, + CapacityConfig: capacityConfig, + } + + req := &managedkafkapb.CreateConnectClusterRequest{ + Parent: locationPath, + ConnectClusterId: clusterID, + ConnectCluster: connectCluster, + } + op, err := client.CreateConnectCluster(ctx, req) + if err != nil { + return fmt.Errorf("client.CreateConnectCluster got err: %w", err) + } + // The duration of this operation can vary considerably, typically taking 5-15 minutes. + resp, err := op.Wait(ctx) + if err != nil { + return fmt.Errorf("op.Wait got err: %w", err) + } + fmt.Fprintf(w, "Created connect cluster: %s\n", resp.Name) + return nil +} + +// [END managedkafka_create_connect_cluster] diff --git a/managedkafka/connect/clusters/delete_connect_cluster.go b/managedkafka/connect/clusters/delete_connect_cluster.go new file mode 100644 index 0000000000..7d0c2ddddd --- /dev/null +++ b/managedkafka/connect/clusters/delete_connect_cluster.go @@ -0,0 +1,56 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package clusters + +// [START managedkafka_delete_connect_cluster] +import ( + "context" + "fmt" + "io" + + "cloud.google.com/go/managedkafka/apiv1/managedkafkapb" + "google.golang.org/api/option" + + managedkafka "cloud.google.com/go/managedkafka/apiv1" +) + +func deleteConnectCluster(w io.Writer, projectID, region, clusterID string, opts ...option.ClientOption) error { + // projectID := "my-project-id" + // region := "us-central1" + // clusterID := "my-connect-cluster" + ctx := context.Background() + client, err := managedkafka.NewManagedKafkaConnectClient(ctx, opts...) + if err != nil { + return fmt.Errorf("managedkafka.NewManagedKafkaConnectClient got err: %w", err) + } + defer client.Close() + + clusterPath := fmt.Sprintf("projects/%s/locations/%s/connectClusters/%s", projectID, region, clusterID) + req := &managedkafkapb.DeleteConnectClusterRequest{ + Name: clusterPath, + } + op, err := client.DeleteConnectCluster(ctx, req) + if err != nil { + return fmt.Errorf("client.DeleteConnectCluster got err: %w", err) + } + err = op.Wait(ctx) + if err != nil { + return fmt.Errorf("op.Wait got err: %w", err) + } + fmt.Fprint(w, "Deleted connect cluster\n") + return nil +} + +// [END managedkafka_delete_connect_cluster] diff --git a/managedkafka/connect/clusters/get_connect_cluster.go b/managedkafka/connect/clusters/get_connect_cluster.go new file mode 100644 index 0000000000..efcb8b3604 --- /dev/null +++ b/managedkafka/connect/clusters/get_connect_cluster.go @@ -0,0 +1,52 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package clusters + +// [START managedkafka_get_connect_cluster] +import ( + "context" + "fmt" + "io" + + "cloud.google.com/go/managedkafka/apiv1/managedkafkapb" + "google.golang.org/api/option" + + managedkafka "cloud.google.com/go/managedkafka/apiv1" +) + +func getConnectCluster(w io.Writer, projectID, region, clusterID string, opts ...option.ClientOption) error { + // projectID := "my-project-id" + // region := "us-central1" + // clusterID := "my-connect-cluster" + ctx := context.Background() + client, err := managedkafka.NewManagedKafkaConnectClient(ctx, opts...) + if err != nil { + return fmt.Errorf("managedkafka.NewManagedKafkaConnectClient got err: %w", err) + } + defer client.Close() + + clusterPath := fmt.Sprintf("projects/%s/locations/%s/connectClusters/%s", projectID, region, clusterID) + req := &managedkafkapb.GetConnectClusterRequest{ + Name: clusterPath, + } + cluster, err := client.GetConnectCluster(ctx, req) + if err != nil { + return fmt.Errorf("client.GetConnectCluster got err: %w", err) + } + fmt.Fprintf(w, "Got connect cluster: %#v\n", cluster) + return nil +} + +// [END managedkafka_get_connect_cluster] diff --git a/managedkafka/connect/clusters/list_connect_clusters.go b/managedkafka/connect/clusters/list_connect_clusters.go new file mode 100644 index 0000000000..b01e6c31b2 --- /dev/null +++ b/managedkafka/connect/clusters/list_connect_clusters.go @@ -0,0 +1,58 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package clusters + +// [START managedkafka_list_connect_clusters] +import ( + "context" + "fmt" + "io" + + "cloud.google.com/go/managedkafka/apiv1/managedkafkapb" + "google.golang.org/api/iterator" + "google.golang.org/api/option" + + managedkafka "cloud.google.com/go/managedkafka/apiv1" +) + +func listConnectClusters(w io.Writer, projectID, region string, opts ...option.ClientOption) error { + // projectID := "my-project-id" + // region := "us-central1" + ctx := context.Background() + client, err := managedkafka.NewManagedKafkaConnectClient(ctx, opts...) + if err != nil { + return fmt.Errorf("managedkafka.NewManagedKafkaConnectClient got err: %w", err) + } + defer client.Close() + + locationPath := fmt.Sprintf("projects/%s/locations/%s", projectID, region) + req := &managedkafkapb.ListConnectClustersRequest{ + Parent: locationPath, + } + clusterIter := client.ListConnectClusters(ctx, req) + for { + res, err := clusterIter.Next() + if err == iterator.Done { + break + } + if err != nil { + return fmt.Errorf("clusterIter.Next() got err: %w", err) + } + fmt.Fprintf(w, "Got connect cluster: %v", res) + } + return nil +} + +// [END managedkafka_list_connect_clusters] diff --git a/managedkafka/connect/clusters/update_connect_cluster.go b/managedkafka/connect/clusters/update_connect_cluster.go new file mode 100644 index 0000000000..649a76812b --- /dev/null +++ b/managedkafka/connect/clusters/update_connect_cluster.go @@ -0,0 +1,76 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package clusters + +// [START managedkafka_update_connect_cluster] +import ( + "context" + "fmt" + "io" + + "cloud.google.com/go/managedkafka/apiv1/managedkafkapb" + "google.golang.org/api/option" + "google.golang.org/protobuf/types/known/fieldmaskpb" + + managedkafka "cloud.google.com/go/managedkafka/apiv1" +) + +func updateConnectCluster(w io.Writer, projectID, region, clusterID string, memoryBytes int64, labels map[string]string, opts ...option.ClientOption) error { + // projectID := "my-project-id" + // region := "us-central1" + // clusterID := "my-connect-cluster" + // memoryBytes := 13958643712 // 13 GiB in bytes + // labels := map[string]string{"environment": "production"} + ctx := context.Background() + client, err := managedkafka.NewManagedKafkaConnectClient(ctx, opts...) + if err != nil { + return fmt.Errorf("managedkafka.NewManagedKafkaConnectClient got err: %w", err) + } + defer client.Close() + + clusterPath := fmt.Sprintf("projects/%s/locations/%s/connectClusters/%s", projectID, region, clusterID) + + // Capacity configuration update + capacityConfig := &managedkafkapb.CapacityConfig{ + MemoryBytes: memoryBytes, + } + + connectCluster := &managedkafkapb.ConnectCluster{ + Name: clusterPath, + CapacityConfig: capacityConfig, + Labels: labels, + } + paths := []string{"capacity_config.memory_bytes", "labels"} + updateMask := &fieldmaskpb.FieldMask{ + Paths: paths, + } + + req := &managedkafkapb.UpdateConnectClusterRequest{ + UpdateMask: updateMask, + ConnectCluster: connectCluster, + } + op, err := client.UpdateConnectCluster(ctx, req) + if err != nil { + return fmt.Errorf("client.UpdateConnectCluster got err: %w", err) + } + resp, err := op.Wait(ctx) + if err != nil { + return fmt.Errorf("op.Wait got err: %w", err) + } + fmt.Fprintf(w, "Updated connect cluster: %#v\n", resp) + return nil +} + +// [END managedkafka_update_connect_cluster] diff --git a/managedkafka/connect/connectors/connectors_test.go b/managedkafka/connect/connectors/connectors_test.go new file mode 100644 index 0000000000..551cc5acc5 --- /dev/null +++ b/managedkafka/connect/connectors/connectors_test.go @@ -0,0 +1,194 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package connectors + +import ( + "bytes" + "fmt" + "strings" + "testing" + "time" + + "github.com/GoogleCloudPlatform/golang-samples/internal/managedkafka/fake" + "github.com/GoogleCloudPlatform/golang-samples/internal/testutil" +) + +const ( + connectorPrefix = "connector" + connectClusterID = "test-connect-cluster" + region = "us-central1" +) + +func TestConnectors(t *testing.T) { + tc := testutil.SystemTest(t) + buf := new(bytes.Buffer) + connectorID := fmt.Sprintf("%s-%d", connectorPrefix, time.Now().UnixNano()) + options := fake.Options(t) + + t.Run("CreateMirrorMakerConnector", func(t *testing.T) { + sourceDNS := "source-cluster-dns:9092" + targetDNS := "target-cluster-dns:9092" + topicName := "test-topic" + if err := createMirrorMakerConnector(buf, tc.ProjectID, region, connectClusterID, connectorID+"-mm2", sourceDNS, targetDNS, topicName, options...); err != nil { + t.Fatalf("failed to create MirrorMaker connector: %v", err) + } + got := buf.String() + want := "Created MirrorMaker connector" + if !strings.Contains(got, want) { + t.Fatalf("createMirrorMakerConnector() mismatch got: %s\nwant: %s", got, want) + } + }) + + t.Run("CreatePubSubSourceConnector", func(t *testing.T) { + topicName := "test-topic" + subscription := "test-subscription" + if err := createPubSubSourceConnector(buf, tc.ProjectID, region, connectClusterID, connectorID+"-pubsub-source", topicName, subscription, options...); err != nil { + t.Fatalf("failed to create Pub/Sub source connector: %v", err) + } + got := buf.String() + want := "Created Pub/Sub source connector" + if !strings.Contains(got, want) { + t.Fatalf("createPubSubSourceConnector() mismatch got: %s\nwant: %s", got, want) + } + }) + + t.Run("CreatePubSubSinkConnector", func(t *testing.T) { + topicName := "test-topic" + pubsubTopic := "test-pubsub-topic" + if err := createPubSubSinkConnector(buf, tc.ProjectID, region, connectClusterID, connectorID+"-pubsub-sink", topicName, pubsubTopic, options...); err != nil { + t.Fatalf("failed to create Pub/Sub sink connector: %v", err) + } + got := buf.String() + want := "Created Pub/Sub sink connector" + if !strings.Contains(got, want) { + t.Fatalf("createPubSubSinkConnector() mismatch got: %s\nwant: %s", got, want) + } + }) + + t.Run("CreateGCSSinkConnector", func(t *testing.T) { + topicName := "test-topic" + bucketName := "test-bucket" + if err := createGCSSinkConnector(buf, tc.ProjectID, region, connectClusterID, connectorID+"-gcs-sink", topicName, bucketName, options...); err != nil { + t.Fatalf("failed to create GCS sink connector: %v", err) + } + got := buf.String() + want := "Created Cloud Storage sink connector" + if !strings.Contains(got, want) { + t.Fatalf("createGCSSinkConnector() mismatch got: %s\nwant: %s", got, want) + } + }) + + t.Run("CreateBigQuerySinkConnector", func(t *testing.T) { + topicName := "test-topic" + datasetID := "test-dataset" + if err := createBigQuerySinkConnector(buf, tc.ProjectID, region, connectClusterID, connectorID+"-bq-sink", topicName, datasetID, options...); err != nil { + t.Fatalf("failed to create BigQuery sink connector: %v", err) + } + got := buf.String() + want := "Created BigQuery sink connector" + if !strings.Contains(got, want) { + t.Fatalf("createBigQuerySinkConnector() mismatch got: %s\nwant: %s", got, want) + } + }) + + t.Run("GetConnector", func(t *testing.T) { + if err := getConnector(buf, tc.ProjectID, region, connectClusterID, connectorID, options...); err != nil { + t.Fatalf("failed to get connector: %v", err) + } + got := buf.String() + want := "Got connector" + if !strings.Contains(got, want) { + t.Fatalf("getConnector() mismatch got: %s\nwant: %s", got, want) + } + }) + + t.Run("UpdateConnector", func(t *testing.T) { + config := map[string]string{"tasks.max": "2"} + if err := updateConnector(buf, tc.ProjectID, region, connectClusterID, connectorID, config, options...); err != nil { + t.Fatalf("failed to update connector: %v", err) + } + got := buf.String() + want := "Updated connector" + if !strings.Contains(got, want) { + t.Fatalf("updateConnector() mismatch got: %s\nwant: %s", got, want) + } + }) + + t.Run("ListConnectors", func(t *testing.T) { + if err := listConnectors(buf, tc.ProjectID, region, connectClusterID, options...); err != nil { + t.Fatalf("failed to list connectors: %v", err) + } + got := buf.String() + want := "Got connector" + if !strings.Contains(got, want) { + t.Fatalf("listConnectors() mismatch got: %s\nwant: %s", got, want) + } + }) + + t.Run("PauseConnector", func(t *testing.T) { + if err := pauseConnector(buf, tc.ProjectID, region, connectClusterID, connectorID, options...); err != nil { + t.Fatalf("failed to pause connector: %v", err) + } + got := buf.String() + want := "Paused connector" + if !strings.Contains(got, want) { + t.Fatalf("pauseConnector() mismatch got: %s\nwant: %s", got, want) + } + }) + + t.Run("ResumeConnector", func(t *testing.T) { + if err := resumeConnector(buf, tc.ProjectID, region, connectClusterID, connectorID, options...); err != nil { + t.Fatalf("failed to resume connector: %v", err) + } + got := buf.String() + want := "Resumed connector" + if !strings.Contains(got, want) { + t.Fatalf("resumeConnector() mismatch got: %s\nwant: %s", got, want) + } + }) + + t.Run("StopConnector", func(t *testing.T) { + if err := stopConnector(buf, tc.ProjectID, region, connectClusterID, connectorID, options...); err != nil { + t.Fatalf("failed to stop connector: %v", err) + } + got := buf.String() + want := "Stopped connector" + if !strings.Contains(got, want) { + t.Fatalf("stopConnector() mismatch got: %s\nwant: %s", got, want) + } + }) + + t.Run("RestartConnector", func(t *testing.T) { + if err := restartConnector(buf, tc.ProjectID, region, connectClusterID, connectorID, options...); err != nil { + t.Fatalf("failed to restart connector: %v", err) + } + got := buf.String() + want := "Restarted connector" + if !strings.Contains(got, want) { + t.Fatalf("restartConnector() mismatch got: %s\nwant: %s", got, want) + } + }) + + t.Run("DeleteConnector", func(t *testing.T) { + if err := deleteConnector(buf, tc.ProjectID, region, connectClusterID, connectorID, options...); err != nil { + t.Fatalf("failed to delete connector: %v", err) + } + got := buf.String() + want := "Deleted connector" + if !strings.Contains(got, want) { + t.Fatalf("deleteConnector() mismatch got: %s\nwant: %s", got, want) + } + }) +} diff --git a/managedkafka/connect/connectors/create_bigquery_sink_connector.go b/managedkafka/connect/connectors/create_bigquery_sink_connector.go new file mode 100644 index 0000000000..dd132e17cb --- /dev/null +++ b/managedkafka/connect/connectors/create_bigquery_sink_connector.go @@ -0,0 +1,77 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package connectors + +// [START managedkafka_create_bigquery_sink_connector] +import ( + "context" + "fmt" + "io" + + "cloud.google.com/go/managedkafka/apiv1/managedkafkapb" + "google.golang.org/api/option" + + managedkafka "cloud.google.com/go/managedkafka/apiv1" +) + +func createBigQuerySinkConnector(w io.Writer, projectID, region, connectClusterID, connectorID, topicName, datasetID string, opts ...option.ClientOption) error { + // projectID := "my-project-id" + // region := "us-central1" + // connectClusterID := "my-connect-cluster" + // connectorID := "my-bigquery-sink-connector" + // topicName := "my-kafka-topic" + // datasetID := "my-bigquery-dataset" + ctx := context.Background() + client, err := managedkafka.NewManagedKafkaConnectClient(ctx, opts...) + if err != nil { + return fmt.Errorf("managedkafka.NewManagedKafkaConnectClient got err: %w", err) + } + defer client.Close() + + parent := fmt.Sprintf("projects/%s/locations/%s/connectClusters/%s", projectID, region, connectClusterID) + + // BigQuery Sink sample connector configuration + config := map[string]string{ + "name": connectorID, + "project": projectID, + "topics": topicName, + "tasks.max": "3", + "connector.class": "com.wepay.kafka.connect.bigquery.BigQuerySinkConnector", + "key.converter": "org.apache.kafka.connect.storage.StringConverter", + "value.converter": "org.apache.kafka.connect.json.JsonConverter", + "value.converter.schemas.enable": "false", + "defaultDataset": datasetID, + } + + connector := &managedkafkapb.Connector{ + Name: fmt.Sprintf("%s/connectors/%s", parent, connectorID), + Configs: config, + } + + req := &managedkafkapb.CreateConnectorRequest{ + Parent: parent, + ConnectorId: connectorID, + Connector: connector, + } + + resp, err := client.CreateConnector(ctx, req) + if err != nil { + return fmt.Errorf("client.CreateConnector got err: %w", err) + } + fmt.Fprintf(w, "Created BigQuery sink connector: %s\n", resp.Name) + return nil +} + +// [END managedkafka_create_bigquery_sink_connector] diff --git a/managedkafka/connect/connectors/create_gcs_sink_connector.go b/managedkafka/connect/connectors/create_gcs_sink_connector.go new file mode 100644 index 0000000000..4594ad8abf --- /dev/null +++ b/managedkafka/connect/connectors/create_gcs_sink_connector.go @@ -0,0 +1,78 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package connectors + +// [START managedkafka_create_gcs_sink_connector] +import ( + "context" + "fmt" + "io" + + "cloud.google.com/go/managedkafka/apiv1/managedkafkapb" + "google.golang.org/api/option" + + managedkafka "cloud.google.com/go/managedkafka/apiv1" +) + +func createGCSSinkConnector(w io.Writer, projectID, region, connectClusterID, connectorID, topicName, bucketName string, opts ...option.ClientOption) error { + // projectID := "my-project-id" + // region := "us-central1" + // connectClusterID := "my-connect-cluster" + // connectorID := "my-gcs-sink-connector" + // topicName := "my-kafka-topic" + // bucketName := "my-gcs-bucket" + ctx := context.Background() + client, err := managedkafka.NewManagedKafkaConnectClient(ctx, opts...) + if err != nil { + return fmt.Errorf("managedkafka.NewManagedKafkaConnectClient got err: %w", err) + } + defer client.Close() + + parent := fmt.Sprintf("projects/%s/locations/%s/connectClusters/%s", projectID, region, connectClusterID) + + // Cloud Storage Sink sample connector configuration + config := map[string]string{ + "connector.class": "io.aiven.kafka.connect.gcs.GcsSinkConnector", + "tasks.max": "1", + "topics": topicName, + "gcs.bucket.name": bucketName, + "gcs.credentials.default": "true", + "format.output.type": "json", + "name": connectorID, + "value.converter": "org.apache.kafka.connect.json.JsonConverter", + "value.converter.schemas.enable": "false", + "key.converter": "org.apache.kafka.connect.storage.StringConverter", + } + + connector := &managedkafkapb.Connector{ + Name: fmt.Sprintf("%s/connectors/%s", parent, connectorID), + Configs: config, + } + + req := &managedkafkapb.CreateConnectorRequest{ + Parent: parent, + ConnectorId: connectorID, + Connector: connector, + } + + resp, err := client.CreateConnector(ctx, req) + if err != nil { + return fmt.Errorf("client.CreateConnector got err: %w", err) + } + fmt.Fprintf(w, "Created Cloud Storage sink connector: %s\n", resp.Name) + return nil +} + +// [END managedkafka_create_gcs_sink_connector] diff --git a/managedkafka/connect/connectors/create_mirrormaker_connector.go b/managedkafka/connect/connectors/create_mirrormaker_connector.go new file mode 100644 index 0000000000..5005e0b95f --- /dev/null +++ b/managedkafka/connect/connectors/create_mirrormaker_connector.go @@ -0,0 +1,85 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package connectors + +// [START managedkafka_create_mirrormaker_connector] +import ( + "context" + "fmt" + "io" + + "cloud.google.com/go/managedkafka/apiv1/managedkafkapb" + "google.golang.org/api/option" + + managedkafka "cloud.google.com/go/managedkafka/apiv1" +) + +func createMirrorMakerConnector(w io.Writer, projectID, region, connectClusterID, connectorID, sourceDNS, targetDNS, topicName string, opts ...option.ClientOption) error { + // projectID := "my-project-id" + // region := "us-central1" + // connectClusterID := "my-connect-cluster" + // connectorID := "my-mm2-connector" + // sourceDNS := "source-cluster-dns:9092" + // targetDNS := "target-cluster-dns:9092" + // topicName := "my-topic" + ctx := context.Background() + client, err := managedkafka.NewManagedKafkaConnectClient(ctx, opts...) + if err != nil { + return fmt.Errorf("managedkafka.NewManagedKafkaConnectClient got err: %w", err) + } + defer client.Close() + + parent := fmt.Sprintf("projects/%s/locations/%s/connectClusters/%s", projectID, region, connectClusterID) + + // MirrorMaker 2.0 sample connector configuration + config := map[string]string{ + "connector.class": "org.apache.kafka.connect.mirror.MirrorSourceConnector", + "name": connectorID, + "source.cluster.alias": "source", + "target.cluster.alias": "target", + "topics": topicName, + "source.cluster.bootstrap.servers": sourceDNS, + "target.cluster.bootstrap.servers": targetDNS, + "offset-syncs.topic.replication.factor": "1", + "source.cluster.security.protocol": "SASL_SSL", + "source.cluster.sasl.mechanism": "OAUTHBEARER", + "source.cluster.sasl.login.callback.handler.class": "com.google.cloud.hosted.kafka.auth.GcpLoginCallbackHandler", + "source.cluster.sasl.jaas.config": "org.apache.kafka.common.security.oauthbearer.OAuthBearerLoginModule required;", + "target.cluster.security.protocol": "SASL_SSL", + "target.cluster.sasl.mechanism": "OAUTHBEARER", + "target.cluster.sasl.login.callback.handler.class": "com.google.cloud.hosted.kafka.auth.GcpLoginCallbackHandler", + "target.cluster.sasl.jaas.config": "org.apache.kafka.common.security.oauthbearer.OAuthBearerLoginModule required;", + } + + connector := &managedkafkapb.Connector{ + Name: fmt.Sprintf("%s/connectors/%s", parent, connectorID), + Configs: config, + } + + req := &managedkafkapb.CreateConnectorRequest{ + Parent: parent, + ConnectorId: connectorID, + Connector: connector, + } + + resp, err := client.CreateConnector(ctx, req) + if err != nil { + return fmt.Errorf("client.CreateConnector got err: %w", err) + } + fmt.Fprintf(w, "Created MirrorMaker connector: %s\n", resp.Name) + return nil +} + +// [END managedkafka_create_mirrormaker_connector] diff --git a/managedkafka/connect/connectors/create_pubsub_sink_connector.go b/managedkafka/connect/connectors/create_pubsub_sink_connector.go new file mode 100644 index 0000000000..7039f959d6 --- /dev/null +++ b/managedkafka/connect/connectors/create_pubsub_sink_connector.go @@ -0,0 +1,76 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package connectors + +// [START managedkafka_create_pubsub_sink_connector] +import ( + "context" + "fmt" + "io" + + "cloud.google.com/go/managedkafka/apiv1/managedkafkapb" + "google.golang.org/api/option" + + managedkafka "cloud.google.com/go/managedkafka/apiv1" +) + +func createPubSubSinkConnector(w io.Writer, projectID, region, connectClusterID, connectorID, topicName, pubsubTopic string, opts ...option.ClientOption) error { + // projectID := "my-project-id" + // region := "us-central1" + // connectClusterID := "my-connect-cluster" + // connectorID := "my-pubsub-sink-connector" + // topicName := "my-kafka-topic" + // pubsubTopic := "my-pubsub-topic" + ctx := context.Background() + client, err := managedkafka.NewManagedKafkaConnectClient(ctx, opts...) + if err != nil { + return fmt.Errorf("managedkafka.NewManagedKafkaConnectClient got err: %w", err) + } + defer client.Close() + + parent := fmt.Sprintf("projects/%s/locations/%s/connectClusters/%s", projectID, region, connectClusterID) + + // Pub/Sub Sink sample connector configuration + config := map[string]string{ + "connector.class": "com.google.pubsub.kafka.sink.CloudPubSubSinkConnector", + "name": connectorID, + "tasks.max": "1", + "topics": topicName, + "value.converter": "org.apache.kafka.connect.storage.StringConverter", + "key.converter": "org.apache.kafka.connect.storage.StringConverter", + "cps.topic": pubsubTopic, + "cps.project": projectID, + } + + connector := &managedkafkapb.Connector{ + Name: fmt.Sprintf("%s/connectors/%s", parent, connectorID), + Configs: config, + } + + req := &managedkafkapb.CreateConnectorRequest{ + Parent: parent, + ConnectorId: connectorID, + Connector: connector, + } + + resp, err := client.CreateConnector(ctx, req) + if err != nil { + return fmt.Errorf("client.CreateConnector got err: %w", err) + } + fmt.Fprintf(w, "Created Pub/Sub sink connector: %s\n", resp.Name) + return nil +} + +// [END managedkafka_create_pubsub_sink_connector] diff --git a/managedkafka/connect/connectors/create_pubsub_source_connector.go b/managedkafka/connect/connectors/create_pubsub_source_connector.go new file mode 100644 index 0000000000..709c5dea8c --- /dev/null +++ b/managedkafka/connect/connectors/create_pubsub_source_connector.go @@ -0,0 +1,76 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package connectors + +// [START managedkafka_create_pubsub_source_connector] +import ( + "context" + "fmt" + "io" + + "cloud.google.com/go/managedkafka/apiv1/managedkafkapb" + "google.golang.org/api/option" + + managedkafka "cloud.google.com/go/managedkafka/apiv1" +) + +func createPubSubSourceConnector(w io.Writer, projectID, region, connectClusterID, connectorID, topicName, subscription string, opts ...option.ClientOption) error { + // projectID := "my-project-id" + // region := "us-central1" + // connectClusterID := "my-connect-cluster" + // connectorID := "my-pubsub-source-connector" + // topicName := "my-kafka-topic" + // subscription := "my-pubsub-subscription" + ctx := context.Background() + client, err := managedkafka.NewManagedKafkaConnectClient(ctx, opts...) + if err != nil { + return fmt.Errorf("managedkafka.NewManagedKafkaConnectClient got err: %w", err) + } + defer client.Close() + + parent := fmt.Sprintf("projects/%s/locations/%s/connectClusters/%s", projectID, region, connectClusterID) + + // Pub/Sub Source sample connector configuration + config := map[string]string{ + "connector.class": "com.google.pubsub.kafka.source.CloudPubSubSourceConnector", + "name": connectorID, + "tasks.max": "1", + "kafka.topic": topicName, + "cps.subscription": subscription, + "cps.project": projectID, + "value.converter": "org.apache.kafka.connect.converters.ByteArrayConverter", + "key.converter": "org.apache.kafka.connect.storage.StringConverter", + } + + connector := &managedkafkapb.Connector{ + Name: fmt.Sprintf("%s/connectors/%s", parent, connectorID), + Configs: config, + } + + req := &managedkafkapb.CreateConnectorRequest{ + Parent: parent, + ConnectorId: connectorID, + Connector: connector, + } + + resp, err := client.CreateConnector(ctx, req) + if err != nil { + return fmt.Errorf("client.CreateConnector got err: %w", err) + } + fmt.Fprintf(w, "Created Pub/Sub source connector: %s\n", resp.Name) + return nil +} + +// [END managedkafka_create_pubsub_source_connector] diff --git a/managedkafka/connect/connectors/delete_connector.go b/managedkafka/connect/connectors/delete_connector.go new file mode 100644 index 0000000000..c6bc13cffb --- /dev/null +++ b/managedkafka/connect/connectors/delete_connector.go @@ -0,0 +1,53 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package connectors + +// [START managedkafka_delete_connector] +import ( + "context" + "fmt" + "io" + + "cloud.google.com/go/managedkafka/apiv1/managedkafkapb" + "google.golang.org/api/option" + + managedkafka "cloud.google.com/go/managedkafka/apiv1" +) + +func deleteConnector(w io.Writer, projectID, region, connectClusterID, connectorID string, opts ...option.ClientOption) error { + // projectID := "my-project-id" + // region := "us-central1" + // connectClusterID := "my-connect-cluster" + // connectorID := "my-connector" + ctx := context.Background() + client, err := managedkafka.NewManagedKafkaConnectClient(ctx, opts...) + if err != nil { + return fmt.Errorf("managedkafka.NewManagedKafkaConnectClient got err: %w", err) + } + defer client.Close() + + connectorPath := fmt.Sprintf("projects/%s/locations/%s/connectClusters/%s/connectors/%s", projectID, region, connectClusterID, connectorID) + req := &managedkafkapb.DeleteConnectorRequest{ + Name: connectorPath, + } + err = client.DeleteConnector(ctx, req) + if err != nil { + return fmt.Errorf("client.DeleteConnector got err: %w", err) + } + fmt.Fprint(w, "Deleted connector\n") + return nil +} + +// [END managedkafka_delete_connector] diff --git a/managedkafka/connect/connectors/get_connector.go b/managedkafka/connect/connectors/get_connector.go new file mode 100644 index 0000000000..3003556667 --- /dev/null +++ b/managedkafka/connect/connectors/get_connector.go @@ -0,0 +1,53 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package connectors + +// [START managedkafka_get_connector] +import ( + "context" + "fmt" + "io" + + "cloud.google.com/go/managedkafka/apiv1/managedkafkapb" + "google.golang.org/api/option" + + managedkafka "cloud.google.com/go/managedkafka/apiv1" +) + +func getConnector(w io.Writer, projectID, region, connectClusterID, connectorID string, opts ...option.ClientOption) error { + // projectID := "my-project-id" + // region := "us-central1" + // connectClusterID := "my-connect-cluster" + // connectorID := "my-connector" + ctx := context.Background() + client, err := managedkafka.NewManagedKafkaConnectClient(ctx, opts...) + if err != nil { + return fmt.Errorf("managedkafka.NewManagedKafkaConnectClient got err: %w", err) + } + defer client.Close() + + connectorPath := fmt.Sprintf("projects/%s/locations/%s/connectClusters/%s/connectors/%s", projectID, region, connectClusterID, connectorID) + req := &managedkafkapb.GetConnectorRequest{ + Name: connectorPath, + } + connector, err := client.GetConnector(ctx, req) + if err != nil { + return fmt.Errorf("client.GetConnector got err: %w", err) + } + fmt.Fprintf(w, "Got connector: %#v\n", connector) + return nil +} + +// [END managedkafka_get_connector] diff --git a/managedkafka/connect/connectors/list_connectors.go b/managedkafka/connect/connectors/list_connectors.go new file mode 100644 index 0000000000..ebea436a26 --- /dev/null +++ b/managedkafka/connect/connectors/list_connectors.go @@ -0,0 +1,59 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package connectors + +// [START managedkafka_list_connectors] +import ( + "context" + "fmt" + "io" + + "cloud.google.com/go/managedkafka/apiv1/managedkafkapb" + "google.golang.org/api/iterator" + "google.golang.org/api/option" + + managedkafka "cloud.google.com/go/managedkafka/apiv1" +) + +func listConnectors(w io.Writer, projectID, region, connectClusterID string, opts ...option.ClientOption) error { + // projectID := "my-project-id" + // region := "us-central1" + // connectClusterID := "my-connect-cluster" + ctx := context.Background() + client, err := managedkafka.NewManagedKafkaConnectClient(ctx, opts...) + if err != nil { + return fmt.Errorf("managedkafka.NewManagedKafkaConnectClient got err: %w", err) + } + defer client.Close() + + parent := fmt.Sprintf("projects/%s/locations/%s/connectClusters/%s", projectID, region, connectClusterID) + req := &managedkafkapb.ListConnectorsRequest{ + Parent: parent, + } + connectorIter := client.ListConnectors(ctx, req) + for { + connector, err := connectorIter.Next() + if err == iterator.Done { + break + } + if err != nil { + return fmt.Errorf("connectorIter.Next() got err: %w", err) + } + fmt.Fprintf(w, "Got connector: %v", connector) + } + return nil +} + +// [END managedkafka_list_connectors] diff --git a/managedkafka/connect/connectors/pause_connector.go b/managedkafka/connect/connectors/pause_connector.go new file mode 100644 index 0000000000..8498e0924c --- /dev/null +++ b/managedkafka/connect/connectors/pause_connector.go @@ -0,0 +1,53 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package connectors + +// [START managedkafka_pause_connector] +import ( + "context" + "fmt" + "io" + + "cloud.google.com/go/managedkafka/apiv1/managedkafkapb" + "google.golang.org/api/option" + + managedkafka "cloud.google.com/go/managedkafka/apiv1" +) + +func pauseConnector(w io.Writer, projectID, region, connectClusterID, connectorID string, opts ...option.ClientOption) error { + // projectID := "my-project-id" + // region := "us-central1" + // connectClusterID := "my-connect-cluster" + // connectorID := "my-connector" + ctx := context.Background() + client, err := managedkafka.NewManagedKafkaConnectClient(ctx, opts...) + if err != nil { + return fmt.Errorf("managedkafka.NewManagedKafkaConnectClient got err: %w", err) + } + defer client.Close() + + connectorPath := fmt.Sprintf("projects/%s/locations/%s/connectClusters/%s/connectors/%s", projectID, region, connectClusterID, connectorID) + req := &managedkafkapb.PauseConnectorRequest{ + Name: connectorPath, + } + resp, err := client.PauseConnector(ctx, req) + if err != nil { + return fmt.Errorf("client.PauseConnector got err: %w", err) + } + fmt.Fprintf(w, "Paused connector: %#v\n", resp) + return nil +} + +// [END managedkafka_pause_connector] diff --git a/managedkafka/connect/connectors/restart_connector.go b/managedkafka/connect/connectors/restart_connector.go new file mode 100644 index 0000000000..9d10e6d73c --- /dev/null +++ b/managedkafka/connect/connectors/restart_connector.go @@ -0,0 +1,53 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package connectors + +// [START managedkafka_restart_connector] +import ( + "context" + "fmt" + "io" + + "cloud.google.com/go/managedkafka/apiv1/managedkafkapb" + "google.golang.org/api/option" + + managedkafka "cloud.google.com/go/managedkafka/apiv1" +) + +func restartConnector(w io.Writer, projectID, region, connectClusterID, connectorID string, opts ...option.ClientOption) error { + // projectID := "my-project-id" + // region := "us-central1" + // connectClusterID := "my-connect-cluster" + // connectorID := "my-connector" + ctx := context.Background() + client, err := managedkafka.NewManagedKafkaConnectClient(ctx, opts...) + if err != nil { + return fmt.Errorf("managedkafka.NewManagedKafkaConnectClient got err: %w", err) + } + defer client.Close() + + connectorPath := fmt.Sprintf("projects/%s/locations/%s/connectClusters/%s/connectors/%s", projectID, region, connectClusterID, connectorID) + req := &managedkafkapb.RestartConnectorRequest{ + Name: connectorPath, + } + resp, err := client.RestartConnector(ctx, req) + if err != nil { + return fmt.Errorf("client.RestartConnector got err: %w", err) + } + fmt.Fprintf(w, "Restarted connector: %#v\n", resp) + return nil +} + +// [END managedkafka_restart_connector] diff --git a/managedkafka/connect/connectors/resume_connector.go b/managedkafka/connect/connectors/resume_connector.go new file mode 100644 index 0000000000..5948eae7db --- /dev/null +++ b/managedkafka/connect/connectors/resume_connector.go @@ -0,0 +1,53 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package connectors + +// [START managedkafka_resume_connector] +import ( + "context" + "fmt" + "io" + + "cloud.google.com/go/managedkafka/apiv1/managedkafkapb" + "google.golang.org/api/option" + + managedkafka "cloud.google.com/go/managedkafka/apiv1" +) + +func resumeConnector(w io.Writer, projectID, region, connectClusterID, connectorID string, opts ...option.ClientOption) error { + // projectID := "my-project-id" + // region := "us-central1" + // connectClusterID := "my-connect-cluster" + // connectorID := "my-connector" + ctx := context.Background() + client, err := managedkafka.NewManagedKafkaConnectClient(ctx, opts...) + if err != nil { + return fmt.Errorf("managedkafka.NewManagedKafkaConnectClient got err: %w", err) + } + defer client.Close() + + connectorPath := fmt.Sprintf("projects/%s/locations/%s/connectClusters/%s/connectors/%s", projectID, region, connectClusterID, connectorID) + req := &managedkafkapb.ResumeConnectorRequest{ + Name: connectorPath, + } + resp, err := client.ResumeConnector(ctx, req) + if err != nil { + return fmt.Errorf("client.ResumeConnector got err: %w", err) + } + fmt.Fprintf(w, "Resumed connector: %#v\n", resp) + return nil +} + +// [END managedkafka_resume_connector] diff --git a/managedkafka/connect/connectors/stop_connector.go b/managedkafka/connect/connectors/stop_connector.go new file mode 100644 index 0000000000..52411f29e9 --- /dev/null +++ b/managedkafka/connect/connectors/stop_connector.go @@ -0,0 +1,53 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package connectors + +// [START managedkafka_stop_connector] +import ( + "context" + "fmt" + "io" + + "cloud.google.com/go/managedkafka/apiv1/managedkafkapb" + "google.golang.org/api/option" + + managedkafka "cloud.google.com/go/managedkafka/apiv1" +) + +func stopConnector(w io.Writer, projectID, region, connectClusterID, connectorID string, opts ...option.ClientOption) error { + // projectID := "my-project-id" + // region := "us-central1" + // connectClusterID := "my-connect-cluster" + // connectorID := "my-connector" + ctx := context.Background() + client, err := managedkafka.NewManagedKafkaConnectClient(ctx, opts...) + if err != nil { + return fmt.Errorf("managedkafka.NewManagedKafkaConnectClient got err: %w", err) + } + defer client.Close() + + connectorPath := fmt.Sprintf("projects/%s/locations/%s/connectClusters/%s/connectors/%s", projectID, region, connectClusterID, connectorID) + req := &managedkafkapb.StopConnectorRequest{ + Name: connectorPath, + } + resp, err := client.StopConnector(ctx, req) + if err != nil { + return fmt.Errorf("client.StopConnector got err: %w", err) + } + fmt.Fprintf(w, "Stopped connector: %#v\n", resp) + return nil +} + +// [END managedkafka_stop_connector] diff --git a/managedkafka/connect/connectors/update_connector.go b/managedkafka/connect/connectors/update_connector.go new file mode 100644 index 0000000000..5f51061379 --- /dev/null +++ b/managedkafka/connect/connectors/update_connector.go @@ -0,0 +1,65 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package connectors + +// [START managedkafka_update_connector] +import ( + "context" + "fmt" + "io" + + "cloud.google.com/go/managedkafka/apiv1/managedkafkapb" + "google.golang.org/api/option" + "google.golang.org/protobuf/types/known/fieldmaskpb" + + managedkafka "cloud.google.com/go/managedkafka/apiv1" +) + +func updateConnector(w io.Writer, projectID, region, connectClusterID, connectorID string, config map[string]string, opts ...option.ClientOption) error { + // projectID := "my-project-id" + // region := "us-central1" + // connectClusterID := "my-connect-cluster" + // connectorID := "my-connector" + // config := map[string]string{"tasks.max": "2"} + ctx := context.Background() + client, err := managedkafka.NewManagedKafkaConnectClient(ctx, opts...) + if err != nil { + return fmt.Errorf("managedkafka.NewManagedKafkaConnectClient got err: %w", err) + } + defer client.Close() + + connectorPath := fmt.Sprintf("projects/%s/locations/%s/connectClusters/%s/connectors/%s", projectID, region, connectClusterID, connectorID) + connector := &managedkafkapb.Connector{ + Name: connectorPath, + Configs: config, + } + paths := []string{"configs"} + updateMask := &fieldmaskpb.FieldMask{ + Paths: paths, + } + + req := &managedkafkapb.UpdateConnectorRequest{ + UpdateMask: updateMask, + Connector: connector, + } + resp, err := client.UpdateConnector(ctx, req) + if err != nil { + return fmt.Errorf("client.UpdateConnector got err: %w", err) + } + fmt.Fprintf(w, "Updated connector: %#v\n", resp) + return nil +} + +// [END managedkafka_update_connector] diff --git a/managedkafka/go.mod b/managedkafka/go.mod index e15f5bcd47..20d881d378 100644 --- a/managedkafka/go.mod +++ b/managedkafka/go.mod @@ -3,59 +3,61 @@ module github.com/GoogleCloudPlatform/golang-samples/managedkafka go 1.23.0 require ( - cloud.google.com/go/managedkafka v0.1.3 + cloud.google.com/go/managedkafka v0.7.0 github.com/GoogleCloudPlatform/golang-samples v0.0.0-20240724083556-7f760db013b7 github.com/GoogleCloudPlatform/golang-samples/internal/managedkafka v0.0.0-00010101000000-000000000000 - google.golang.org/api v0.217.0 - google.golang.org/protobuf v1.36.3 + google.golang.org/api v0.239.0 + google.golang.org/protobuf v1.36.6 ) require ( - cel.dev/expr v0.19.1 // indirect - cloud.google.com/go v0.118.0 // indirect - cloud.google.com/go/auth v0.14.0 // indirect - cloud.google.com/go/auth/oauth2adapt v0.2.7 // indirect - cloud.google.com/go/compute/metadata v0.6.0 // indirect - cloud.google.com/go/iam v1.3.1 // indirect - cloud.google.com/go/longrunning v0.6.4 // indirect - cloud.google.com/go/monitoring v1.23.0 // indirect + cel.dev/expr v0.23.0 // indirect + cloud.google.com/go v0.120.0 // indirect + cloud.google.com/go/auth v0.16.2 // indirect + cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect + cloud.google.com/go/compute/metadata v0.7.0 // indirect + cloud.google.com/go/iam v1.5.2 // indirect + cloud.google.com/go/longrunning v0.6.7 // indirect + cloud.google.com/go/monitoring v1.24.2 // indirect cloud.google.com/go/storage v1.50.0 // indirect - github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.25.0 // indirect - github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.49.0 // indirect - github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.49.0 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.27.0 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.50.0 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.50.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect - github.com/cncf/xds/go v0.0.0-20250121191232-2f005788dc42 // indirect - github.com/envoyproxy/go-control-plane/envoy v1.32.3 // indirect - github.com/envoyproxy/protoc-gen-validate v1.1.0 // indirect + github.com/cncf/xds/go v0.0.0-20250326154945-ae57f3c0d45f // indirect + github.com/envoyproxy/go-control-plane/envoy v1.32.4 // indirect + github.com/envoyproxy/protoc-gen-validate v1.2.1 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect + github.com/go-jose/go-jose/v4 v4.0.5 // indirect github.com/go-logr/logr v1.4.2 // indirect github.com/go-logr/stdr v1.2.2 // indirect - github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect github.com/google/s2a-go v0.1.9 // indirect github.com/google/uuid v1.6.0 // indirect - github.com/googleapis/enterprise-certificate-proxy v0.3.4 // indirect - github.com/googleapis/gax-go/v2 v2.14.1 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.3.6 // indirect + github.com/googleapis/gax-go/v2 v2.14.2 // indirect github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect + github.com/spiffe/go-spiffe/v2 v2.5.0 // indirect + github.com/zeebo/errs v1.4.0 // indirect go.opentelemetry.io/auto/sdk v1.1.0 // indirect - go.opentelemetry.io/contrib/detectors/gcp v1.34.0 // indirect - go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.59.0 // indirect - go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.59.0 // indirect - go.opentelemetry.io/otel v1.34.0 // indirect - go.opentelemetry.io/otel/metric v1.34.0 // indirect - go.opentelemetry.io/otel/sdk v1.34.0 // indirect - go.opentelemetry.io/otel/sdk/metric v1.34.0 // indirect - go.opentelemetry.io/otel/trace v1.34.0 // indirect - golang.org/x/crypto v0.32.0 // indirect - golang.org/x/net v0.34.0 // indirect - golang.org/x/oauth2 v0.25.0 // indirect - golang.org/x/sync v0.10.0 // indirect - golang.org/x/sys v0.29.0 // indirect - golang.org/x/text v0.21.0 // indirect - golang.org/x/time v0.9.0 // indirect - google.golang.org/genproto v0.0.0-20250115164207-1a7da9e5054f // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20250115164207-1a7da9e5054f // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20250115164207-1a7da9e5054f // indirect - google.golang.org/grpc v1.69.4 // indirect + go.opentelemetry.io/contrib/detectors/gcp v1.35.0 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.61.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 // indirect + go.opentelemetry.io/otel v1.36.0 // indirect + go.opentelemetry.io/otel/metric v1.36.0 // indirect + go.opentelemetry.io/otel/sdk v1.36.0 // indirect + go.opentelemetry.io/otel/sdk/metric v1.36.0 // indirect + go.opentelemetry.io/otel/trace v1.36.0 // indirect + golang.org/x/crypto v0.39.0 // indirect + golang.org/x/net v0.41.0 // indirect + golang.org/x/oauth2 v0.30.0 // indirect + golang.org/x/sync v0.15.0 // indirect + golang.org/x/sys v0.33.0 // indirect + golang.org/x/text v0.26.0 // indirect + golang.org/x/time v0.12.0 // indirect + google.golang.org/genproto v0.0.0-20250505200425-f936aa4a68b2 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20250603155806-513f23925822 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20250603155806-513f23925822 // indirect + google.golang.org/grpc v1.73.0 // indirect ) replace github.com/GoogleCloudPlatform/golang-samples/internal/managedkafka => ../internal/managedkafka/ From 7c0f5a857c3f79e8915217d43a3aa15870c5cdec Mon Sep 17 00:00:00 2001 From: salmany Date: Mon, 4 Aug 2025 22:43:10 +0000 Subject: [PATCH 02/11] Updated headers. --- managedkafka/connect/connectors/connectors_test.go | 2 +- .../connect/connectors/create_bigquery_sink_connector.go | 2 +- managedkafka/connect/connectors/create_gcs_sink_connector.go | 2 +- managedkafka/connect/connectors/create_mirrormaker_connector.go | 2 +- managedkafka/connect/connectors/create_pubsub_sink_connector.go | 2 +- .../connect/connectors/create_pubsub_source_connector.go | 2 +- managedkafka/connect/connectors/delete_connector.go | 2 +- managedkafka/connect/connectors/get_connector.go | 2 +- managedkafka/connect/connectors/list_connectors.go | 2 +- managedkafka/connect/connectors/pause_connector.go | 2 +- managedkafka/connect/connectors/restart_connector.go | 2 +- managedkafka/connect/connectors/resume_connector.go | 2 +- managedkafka/connect/connectors/stop_connector.go | 2 +- managedkafka/connect/connectors/update_connector.go | 2 +- 14 files changed, 14 insertions(+), 14 deletions(-) diff --git a/managedkafka/connect/connectors/connectors_test.go b/managedkafka/connect/connectors/connectors_test.go index 551cc5acc5..22d622e0b9 100644 --- a/managedkafka/connect/connectors/connectors_test.go +++ b/managedkafka/connect/connectors/connectors_test.go @@ -1,4 +1,4 @@ -// Copyright 2024 Google LLC +// Copyright 2025 Google LLC // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/managedkafka/connect/connectors/create_bigquery_sink_connector.go b/managedkafka/connect/connectors/create_bigquery_sink_connector.go index dd132e17cb..693642bb98 100644 --- a/managedkafka/connect/connectors/create_bigquery_sink_connector.go +++ b/managedkafka/connect/connectors/create_bigquery_sink_connector.go @@ -1,4 +1,4 @@ -// Copyright 2024 Google LLC +// Copyright 2025 Google LLC // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/managedkafka/connect/connectors/create_gcs_sink_connector.go b/managedkafka/connect/connectors/create_gcs_sink_connector.go index 4594ad8abf..c85260795f 100644 --- a/managedkafka/connect/connectors/create_gcs_sink_connector.go +++ b/managedkafka/connect/connectors/create_gcs_sink_connector.go @@ -1,4 +1,4 @@ -// Copyright 2024 Google LLC +// Copyright 2025 Google LLC // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/managedkafka/connect/connectors/create_mirrormaker_connector.go b/managedkafka/connect/connectors/create_mirrormaker_connector.go index 5005e0b95f..06e82d53ed 100644 --- a/managedkafka/connect/connectors/create_mirrormaker_connector.go +++ b/managedkafka/connect/connectors/create_mirrormaker_connector.go @@ -1,4 +1,4 @@ -// Copyright 2024 Google LLC +// Copyright 2025 Google LLC // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/managedkafka/connect/connectors/create_pubsub_sink_connector.go b/managedkafka/connect/connectors/create_pubsub_sink_connector.go index 7039f959d6..745e6243be 100644 --- a/managedkafka/connect/connectors/create_pubsub_sink_connector.go +++ b/managedkafka/connect/connectors/create_pubsub_sink_connector.go @@ -1,4 +1,4 @@ -// Copyright 2024 Google LLC +// Copyright 2025 Google LLC // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/managedkafka/connect/connectors/create_pubsub_source_connector.go b/managedkafka/connect/connectors/create_pubsub_source_connector.go index 709c5dea8c..b3717fd7fc 100644 --- a/managedkafka/connect/connectors/create_pubsub_source_connector.go +++ b/managedkafka/connect/connectors/create_pubsub_source_connector.go @@ -1,4 +1,4 @@ -// Copyright 2024 Google LLC +// Copyright 2025 Google LLC // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/managedkafka/connect/connectors/delete_connector.go b/managedkafka/connect/connectors/delete_connector.go index c6bc13cffb..f0f815c09a 100644 --- a/managedkafka/connect/connectors/delete_connector.go +++ b/managedkafka/connect/connectors/delete_connector.go @@ -1,4 +1,4 @@ -// Copyright 2024 Google LLC +// Copyright 2025 Google LLC // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/managedkafka/connect/connectors/get_connector.go b/managedkafka/connect/connectors/get_connector.go index 3003556667..487afea14f 100644 --- a/managedkafka/connect/connectors/get_connector.go +++ b/managedkafka/connect/connectors/get_connector.go @@ -1,4 +1,4 @@ -// Copyright 2024 Google LLC +// Copyright 2025 Google LLC // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/managedkafka/connect/connectors/list_connectors.go b/managedkafka/connect/connectors/list_connectors.go index ebea436a26..6661e0f720 100644 --- a/managedkafka/connect/connectors/list_connectors.go +++ b/managedkafka/connect/connectors/list_connectors.go @@ -1,4 +1,4 @@ -// Copyright 2024 Google LLC +// Copyright 2025 Google LLC // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/managedkafka/connect/connectors/pause_connector.go b/managedkafka/connect/connectors/pause_connector.go index 8498e0924c..26e58e2528 100644 --- a/managedkafka/connect/connectors/pause_connector.go +++ b/managedkafka/connect/connectors/pause_connector.go @@ -1,4 +1,4 @@ -// Copyright 2024 Google LLC +// Copyright 2025 Google LLC // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/managedkafka/connect/connectors/restart_connector.go b/managedkafka/connect/connectors/restart_connector.go index 9d10e6d73c..3e9b0201ea 100644 --- a/managedkafka/connect/connectors/restart_connector.go +++ b/managedkafka/connect/connectors/restart_connector.go @@ -1,4 +1,4 @@ -// Copyright 2024 Google LLC +// Copyright 2025 Google LLC // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/managedkafka/connect/connectors/resume_connector.go b/managedkafka/connect/connectors/resume_connector.go index 5948eae7db..75f62f2e4a 100644 --- a/managedkafka/connect/connectors/resume_connector.go +++ b/managedkafka/connect/connectors/resume_connector.go @@ -1,4 +1,4 @@ -// Copyright 2024 Google LLC +// Copyright 2025 Google LLC // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/managedkafka/connect/connectors/stop_connector.go b/managedkafka/connect/connectors/stop_connector.go index 52411f29e9..e0ca778bfe 100644 --- a/managedkafka/connect/connectors/stop_connector.go +++ b/managedkafka/connect/connectors/stop_connector.go @@ -1,4 +1,4 @@ -// Copyright 2024 Google LLC +// Copyright 2025 Google LLC // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/managedkafka/connect/connectors/update_connector.go b/managedkafka/connect/connectors/update_connector.go index 5f51061379..6300c5b12c 100644 --- a/managedkafka/connect/connectors/update_connector.go +++ b/managedkafka/connect/connectors/update_connector.go @@ -1,4 +1,4 @@ -// Copyright 2024 Google LLC +// Copyright 2025 Google LLC // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. From 8a63cc4722ffb9dd7047dd8f7a2101276f8cf167 Mon Sep 17 00:00:00 2001 From: salmany Date: Mon, 4 Aug 2025 22:50:59 +0000 Subject: [PATCH 03/11] Reset buffer for sub-tests. --- .../connect/clusters/connect_clusters_test.go | 5 +++++ managedkafka/connect/connectors/connectors_test.go | 13 +++++++++++++ 2 files changed, 18 insertions(+) diff --git a/managedkafka/connect/clusters/connect_clusters_test.go b/managedkafka/connect/clusters/connect_clusters_test.go index 422eee7811..7000276048 100644 --- a/managedkafka/connect/clusters/connect_clusters_test.go +++ b/managedkafka/connect/clusters/connect_clusters_test.go @@ -41,6 +41,7 @@ func TestConnectClusters(t *testing.T) { kafkaClusterPath := fmt.Sprintf("projects/%s/locations/%s/clusters/%s", tc.ProjectID, region, kafkaClusterID) t.Run("CreateConnectCluster", func(t *testing.T) { + buf.Reset() if err := createConnectCluster(buf, tc.ProjectID, region, connectClusterID, kafkaClusterPath, options...); err != nil { t.Fatalf("failed to create a connect cluster: %v", err) } @@ -51,6 +52,7 @@ func TestConnectClusters(t *testing.T) { } }) t.Run("GetConnectCluster", func(t *testing.T) { + buf.Reset() if err := getConnectCluster(buf, tc.ProjectID, region, connectClusterID, options...); err != nil { t.Fatalf("failed to get connect cluster: %v", err) } @@ -61,6 +63,7 @@ func TestConnectClusters(t *testing.T) { } }) t.Run("UpdateConnectCluster", func(t *testing.T) { + buf.Reset() memoryBytes := int64(13958643712) // 13 GiB in bytes labels := map[string]string{"environment": "test"} if err := updateConnectCluster(buf, tc.ProjectID, region, connectClusterID, memoryBytes, labels, options...); err != nil { @@ -73,6 +76,7 @@ func TestConnectClusters(t *testing.T) { } }) t.Run("ListConnectClusters", func(t *testing.T) { + buf.Reset() if err := listConnectClusters(buf, tc.ProjectID, region, options...); err != nil { t.Fatalf("failed to list connect clusters: %v", err) } @@ -83,6 +87,7 @@ func TestConnectClusters(t *testing.T) { } }) t.Run("DeleteConnectCluster", func(t *testing.T) { + buf.Reset() if err := deleteConnectCluster(buf, tc.ProjectID, region, connectClusterID, options...); err != nil { t.Fatalf("failed to delete connect cluster: %v", err) } diff --git a/managedkafka/connect/connectors/connectors_test.go b/managedkafka/connect/connectors/connectors_test.go index 22d622e0b9..132d74fd3b 100644 --- a/managedkafka/connect/connectors/connectors_test.go +++ b/managedkafka/connect/connectors/connectors_test.go @@ -38,6 +38,7 @@ func TestConnectors(t *testing.T) { options := fake.Options(t) t.Run("CreateMirrorMakerConnector", func(t *testing.T) { + buf.Reset() sourceDNS := "source-cluster-dns:9092" targetDNS := "target-cluster-dns:9092" topicName := "test-topic" @@ -52,6 +53,7 @@ func TestConnectors(t *testing.T) { }) t.Run("CreatePubSubSourceConnector", func(t *testing.T) { + buf.Reset() topicName := "test-topic" subscription := "test-subscription" if err := createPubSubSourceConnector(buf, tc.ProjectID, region, connectClusterID, connectorID+"-pubsub-source", topicName, subscription, options...); err != nil { @@ -65,6 +67,7 @@ func TestConnectors(t *testing.T) { }) t.Run("CreatePubSubSinkConnector", func(t *testing.T) { + buf.Reset() topicName := "test-topic" pubsubTopic := "test-pubsub-topic" if err := createPubSubSinkConnector(buf, tc.ProjectID, region, connectClusterID, connectorID+"-pubsub-sink", topicName, pubsubTopic, options...); err != nil { @@ -78,6 +81,7 @@ func TestConnectors(t *testing.T) { }) t.Run("CreateGCSSinkConnector", func(t *testing.T) { + buf.Reset() topicName := "test-topic" bucketName := "test-bucket" if err := createGCSSinkConnector(buf, tc.ProjectID, region, connectClusterID, connectorID+"-gcs-sink", topicName, bucketName, options...); err != nil { @@ -91,6 +95,7 @@ func TestConnectors(t *testing.T) { }) t.Run("CreateBigQuerySinkConnector", func(t *testing.T) { + buf.Reset() topicName := "test-topic" datasetID := "test-dataset" if err := createBigQuerySinkConnector(buf, tc.ProjectID, region, connectClusterID, connectorID+"-bq-sink", topicName, datasetID, options...); err != nil { @@ -104,6 +109,7 @@ func TestConnectors(t *testing.T) { }) t.Run("GetConnector", func(t *testing.T) { + buf.Reset() if err := getConnector(buf, tc.ProjectID, region, connectClusterID, connectorID, options...); err != nil { t.Fatalf("failed to get connector: %v", err) } @@ -115,6 +121,7 @@ func TestConnectors(t *testing.T) { }) t.Run("UpdateConnector", func(t *testing.T) { + buf.Reset() config := map[string]string{"tasks.max": "2"} if err := updateConnector(buf, tc.ProjectID, region, connectClusterID, connectorID, config, options...); err != nil { t.Fatalf("failed to update connector: %v", err) @@ -127,6 +134,7 @@ func TestConnectors(t *testing.T) { }) t.Run("ListConnectors", func(t *testing.T) { + buf.Reset() if err := listConnectors(buf, tc.ProjectID, region, connectClusterID, options...); err != nil { t.Fatalf("failed to list connectors: %v", err) } @@ -138,6 +146,7 @@ func TestConnectors(t *testing.T) { }) t.Run("PauseConnector", func(t *testing.T) { + buf.Reset() if err := pauseConnector(buf, tc.ProjectID, region, connectClusterID, connectorID, options...); err != nil { t.Fatalf("failed to pause connector: %v", err) } @@ -149,6 +158,7 @@ func TestConnectors(t *testing.T) { }) t.Run("ResumeConnector", func(t *testing.T) { + buf.Reset() if err := resumeConnector(buf, tc.ProjectID, region, connectClusterID, connectorID, options...); err != nil { t.Fatalf("failed to resume connector: %v", err) } @@ -160,6 +170,7 @@ func TestConnectors(t *testing.T) { }) t.Run("StopConnector", func(t *testing.T) { + buf.Reset() if err := stopConnector(buf, tc.ProjectID, region, connectClusterID, connectorID, options...); err != nil { t.Fatalf("failed to stop connector: %v", err) } @@ -171,6 +182,7 @@ func TestConnectors(t *testing.T) { }) t.Run("RestartConnector", func(t *testing.T) { + buf.Reset() if err := restartConnector(buf, tc.ProjectID, region, connectClusterID, connectorID, options...); err != nil { t.Fatalf("failed to restart connector: %v", err) } @@ -182,6 +194,7 @@ func TestConnectors(t *testing.T) { }) t.Run("DeleteConnector", func(t *testing.T) { + buf.Reset() if err := deleteConnector(buf, tc.ProjectID, region, connectClusterID, connectorID, options...); err != nil { t.Fatalf("failed to delete connector: %v", err) } From ece4a29ea4b05f6361fc01bcd056999a68a78fa9 Mon Sep 17 00:00:00 2001 From: salmany Date: Tue, 9 Sep 2025 16:11:30 +0000 Subject: [PATCH 04/11] Align code samples with TF / Python / Java samples. --- .../connect/connectors/connectors_test.go | 65 ++++++++++++------- .../create_bigquery_sink_connector.go | 29 +++++---- ...=> create_cloud_storage_sink_connector.go} | 37 ++++++----- .../create_mirrormaker_connector.go | 57 ++++++++-------- .../create_pubsub_sink_connector.go | 31 +++++---- .../create_pubsub_source_connector.go | 30 +++++---- 6 files changed, 146 insertions(+), 103 deletions(-) rename managedkafka/connect/connectors/{create_gcs_sink_connector.go => create_cloud_storage_sink_connector.go} (60%) diff --git a/managedkafka/connect/connectors/connectors_test.go b/managedkafka/connect/connectors/connectors_test.go index 132d74fd3b..9699225f79 100644 --- a/managedkafka/connect/connectors/connectors_test.go +++ b/managedkafka/connect/connectors/connectors_test.go @@ -37,26 +37,33 @@ func TestConnectors(t *testing.T) { connectorID := fmt.Sprintf("%s-%d", connectorPrefix, time.Now().UnixNano()) options := fake.Options(t) - t.Run("CreateMirrorMakerConnector", func(t *testing.T) { + t.Run("CreateMirrorMaker2SourceConnector", func(t *testing.T) { buf.Reset() - sourceDNS := "source-cluster-dns:9092" - targetDNS := "target-cluster-dns:9092" - topicName := "test-topic" - if err := createMirrorMakerConnector(buf, tc.ProjectID, region, connectClusterID, connectorID+"-mm2", sourceDNS, targetDNS, topicName, options...); err != nil { - t.Fatalf("failed to create MirrorMaker connector: %v", err) + sourceBootstrapServers := "source-dns:9092" + targetBootstrapServers := "target-dns:9092" + tasksMax := "3" + sourceClusterAlias := "source" + targetClusterAlias := "target" + topics := ".*" + topicsExclude := "mm2.*.internal,.*.replica,__.*" + if err := createMirrorMaker2SourceConnector(buf, tc.ProjectID, region, connectClusterID, connectorID+"-mm2", sourceBootstrapServers, targetBootstrapServers, tasksMax, sourceClusterAlias, targetClusterAlias, topics, topicsExclude, options...); err != nil { + t.Fatalf("failed to create MirrorMaker 2.0 source connector: %v", err) } got := buf.String() - want := "Created MirrorMaker connector" + want := "Created MirrorMaker 2.0 Source connector" if !strings.Contains(got, want) { - t.Fatalf("createMirrorMakerConnector() mismatch got: %s\nwant: %s", got, want) + t.Fatalf("createMirrorMaker2SourceConnector() mismatch got: %s\nwant: %s", got, want) } }) t.Run("CreatePubSubSourceConnector", func(t *testing.T) { buf.Reset() - topicName := "test-topic" - subscription := "test-subscription" - if err := createPubSubSourceConnector(buf, tc.ProjectID, region, connectClusterID, connectorID+"-pubsub-source", topicName, subscription, options...); err != nil { + kafkaTopic := "test-topic" + cpsSubscription := "test-subscription" + tasksMax := "3" + valueConverter := "org.apache.kafka.connect.converters.ByteArrayConverter" + keyConverter := "org.apache.kafka.connect.storage.StringConverter" + if err := createPubSubSourceConnector(buf, tc.ProjectID, region, connectClusterID, connectorID+"-pubsub-source", kafkaTopic, cpsSubscription, tc.ProjectID, tasksMax, valueConverter, keyConverter, options...); err != nil { t.Fatalf("failed to create Pub/Sub source connector: %v", err) } got := buf.String() @@ -68,9 +75,12 @@ func TestConnectors(t *testing.T) { t.Run("CreatePubSubSinkConnector", func(t *testing.T) { buf.Reset() - topicName := "test-topic" - pubsubTopic := "test-pubsub-topic" - if err := createPubSubSinkConnector(buf, tc.ProjectID, region, connectClusterID, connectorID+"-pubsub-sink", topicName, pubsubTopic, options...); err != nil { + topics := "test-topic" + valueConverter := "org.apache.kafka.connect.storage.StringConverter" + keyConverter := "org.apache.kafka.connect.storage.StringConverter" + cpsTopic := "test-pubsub-topic" + tasksMax := "3" + if err := createPubSubSinkConnector(buf, tc.ProjectID, region, connectClusterID, connectorID+"-pubsub-sink", topics, valueConverter, keyConverter, cpsTopic, tc.ProjectID, tasksMax, options...); err != nil { t.Fatalf("failed to create Pub/Sub sink connector: %v", err) } got := buf.String() @@ -80,25 +90,34 @@ func TestConnectors(t *testing.T) { } }) - t.Run("CreateGCSSinkConnector", func(t *testing.T) { + t.Run("CreateCloudStorageSinkConnector", func(t *testing.T) { buf.Reset() - topicName := "test-topic" - bucketName := "test-bucket" - if err := createGCSSinkConnector(buf, tc.ProjectID, region, connectClusterID, connectorID+"-gcs-sink", topicName, bucketName, options...); err != nil { - t.Fatalf("failed to create GCS sink connector: %v", err) + topics := "test-topic" + gcsBucketName := "test-bucket" + tasksMax := "3" + formatOutputType := "json" + valueConverter := "org.apache.kafka.connect.json.JsonConverter" + valueConverterSchemasEnable := "false" + keyConverter := "org.apache.kafka.connect.storage.StringConverter" + if err := createCloudStorageSinkConnector(buf, tc.ProjectID, region, connectClusterID, connectorID+"-gcs-sink", topics, gcsBucketName, tasksMax, formatOutputType, valueConverter, valueConverterSchemasEnable, keyConverter, options...); err != nil { + t.Fatalf("failed to create Cloud Storage sink connector: %v", err) } got := buf.String() want := "Created Cloud Storage sink connector" if !strings.Contains(got, want) { - t.Fatalf("createGCSSinkConnector() mismatch got: %s\nwant: %s", got, want) + t.Fatalf("createCloudStorageSinkConnector() mismatch got: %s\nwant: %s", got, want) } }) t.Run("CreateBigQuerySinkConnector", func(t *testing.T) { buf.Reset() - topicName := "test-topic" - datasetID := "test-dataset" - if err := createBigQuerySinkConnector(buf, tc.ProjectID, region, connectClusterID, connectorID+"-bq-sink", topicName, datasetID, options...); err != nil { + topics := "test-topic" + tasksMax := "3" + keyConverter := "org.apache.kafka.connect.storage.StringConverter" + valueConverter := "org.apache.kafka.connect.json.JsonConverter" + valueConverterSchemasEnable := "false" + defaultDataset := "test-dataset" + if err := createBigQuerySinkConnector(buf, tc.ProjectID, region, connectClusterID, connectorID+"-bq-sink", topics, tasksMax, keyConverter, valueConverter, valueConverterSchemasEnable, defaultDataset, options...); err != nil { t.Fatalf("failed to create BigQuery sink connector: %v", err) } got := buf.String() diff --git a/managedkafka/connect/connectors/create_bigquery_sink_connector.go b/managedkafka/connect/connectors/create_bigquery_sink_connector.go index 693642bb98..116437af88 100644 --- a/managedkafka/connect/connectors/create_bigquery_sink_connector.go +++ b/managedkafka/connect/connectors/create_bigquery_sink_connector.go @@ -20,19 +20,24 @@ import ( "fmt" "io" + managedkafka "cloud.google.com/go/managedkafka/apiv1" "cloud.google.com/go/managedkafka/apiv1/managedkafkapb" "google.golang.org/api/option" - - managedkafka "cloud.google.com/go/managedkafka/apiv1" ) -func createBigQuerySinkConnector(w io.Writer, projectID, region, connectClusterID, connectorID, topicName, datasetID string, opts ...option.ClientOption) error { +// createBigQuerySinkConnector creates a BigQuery Sink connector. +func createBigQuerySinkConnector(w io.Writer, projectID, region, connectClusterID, connectorID, topics, tasksMax, keyConverter, valueConverter, valueConverterSchemasEnable, defaultDataset string, opts ...option.ClientOption) error { + // TODO(developer): Update with your config values. Here is a sample configuration: // projectID := "my-project-id" // region := "us-central1" // connectClusterID := "my-connect-cluster" - // connectorID := "my-bigquery-sink-connector" - // topicName := "my-kafka-topic" - // datasetID := "my-bigquery-dataset" + // connectorID := "BQ_SINK_CONNECTOR_ID" + // topics := "GMK_TOPIC_ID" + // tasksMax := "3" + // keyConverter := "org.apache.kafka.connect.storage.StringConverter" + // valueConverter := "org.apache.kafka.connect.json.JsonConverter" + // valueConverterSchemasEnable := "false" + // defaultDataset := "BQ_DATASET_ID" ctx := context.Background() client, err := managedkafka.NewManagedKafkaConnectClient(ctx, opts...) if err != nil { @@ -46,13 +51,13 @@ func createBigQuerySinkConnector(w io.Writer, projectID, region, connectClusterI config := map[string]string{ "name": connectorID, "project": projectID, - "topics": topicName, - "tasks.max": "3", + "topics": topics, + "tasks.max": tasksMax, "connector.class": "com.wepay.kafka.connect.bigquery.BigQuerySinkConnector", - "key.converter": "org.apache.kafka.connect.storage.StringConverter", - "value.converter": "org.apache.kafka.connect.json.JsonConverter", - "value.converter.schemas.enable": "false", - "defaultDataset": datasetID, + "key.converter": keyConverter, + "value.converter": valueConverter, + "value.converter.schemas.enable": valueConverterSchemasEnable, + "defaultDataset": defaultDataset, } connector := &managedkafkapb.Connector{ diff --git a/managedkafka/connect/connectors/create_gcs_sink_connector.go b/managedkafka/connect/connectors/create_cloud_storage_sink_connector.go similarity index 60% rename from managedkafka/connect/connectors/create_gcs_sink_connector.go rename to managedkafka/connect/connectors/create_cloud_storage_sink_connector.go index c85260795f..ab2a42ba60 100644 --- a/managedkafka/connect/connectors/create_gcs_sink_connector.go +++ b/managedkafka/connect/connectors/create_cloud_storage_sink_connector.go @@ -14,25 +14,31 @@ package connectors -// [START managedkafka_create_gcs_sink_connector] +// [START managedkafka_create_cloud_storage_sink_connector] import ( "context" "fmt" "io" + managedkafka "cloud.google.com/go/managedkafka/apiv1" "cloud.google.com/go/managedkafka/apiv1/managedkafkapb" "google.golang.org/api/option" - - managedkafka "cloud.google.com/go/managedkafka/apiv1" ) -func createGCSSinkConnector(w io.Writer, projectID, region, connectClusterID, connectorID, topicName, bucketName string, opts ...option.ClientOption) error { +// createCloudStorageSinkConnector creates a Cloud Storage Sink connector. +func createCloudStorageSinkConnector(w io.Writer, projectID, region, connectClusterID, connectorID, topics, gcsBucketName, tasksMax, formatOutputType, valueConverter, valueConverterSchemasEnable, keyConverter string, opts ...option.ClientOption) error { + // TODO(developer): Update with your config values. Here is a sample configuration: // projectID := "my-project-id" // region := "us-central1" // connectClusterID := "my-connect-cluster" - // connectorID := "my-gcs-sink-connector" - // topicName := "my-kafka-topic" - // bucketName := "my-gcs-bucket" + // connectorID := "GCS_SINK_CONNECTOR_ID" + // topics := "GMK_TOPIC_ID" + // gcsBucketName := "GCS_BUCKET_NAME" + // tasksMax := "3" + // formatOutputType := "json" + // valueConverter := "org.apache.kafka.connect.json.JsonConverter" + // valueConverterSchemasEnable := "false" + // keyConverter := "org.apache.kafka.connect.storage.StringConverter" ctx := context.Background() client, err := managedkafka.NewManagedKafkaConnectClient(ctx, opts...) if err != nil { @@ -42,18 +48,17 @@ func createGCSSinkConnector(w io.Writer, projectID, region, connectClusterID, co parent := fmt.Sprintf("projects/%s/locations/%s/connectClusters/%s", projectID, region, connectClusterID) - // Cloud Storage Sink sample connector configuration config := map[string]string{ "connector.class": "io.aiven.kafka.connect.gcs.GcsSinkConnector", - "tasks.max": "1", - "topics": topicName, - "gcs.bucket.name": bucketName, + "tasks.max": tasksMax, + "topics": topics, + "gcs.bucket.name": gcsBucketName, "gcs.credentials.default": "true", - "format.output.type": "json", + "format.output.type": formatOutputType, "name": connectorID, - "value.converter": "org.apache.kafka.connect.json.JsonConverter", - "value.converter.schemas.enable": "false", - "key.converter": "org.apache.kafka.connect.storage.StringConverter", + "value.converter": valueConverter, + "value.converter.schemas.enable": valueConverterSchemasEnable, + "key.converter": keyConverter, } connector := &managedkafkapb.Connector{ @@ -75,4 +80,4 @@ func createGCSSinkConnector(w io.Writer, projectID, region, connectClusterID, co return nil } -// [END managedkafka_create_gcs_sink_connector] +// [END managedkafka_create_cloud_storage_sink_connector] diff --git a/managedkafka/connect/connectors/create_mirrormaker_connector.go b/managedkafka/connect/connectors/create_mirrormaker_connector.go index 06e82d53ed..b7cecaa9f8 100644 --- a/managedkafka/connect/connectors/create_mirrormaker_connector.go +++ b/managedkafka/connect/connectors/create_mirrormaker_connector.go @@ -14,26 +14,31 @@ package connectors -// [START managedkafka_create_mirrormaker_connector] +// [START managedkafka_create_mirrormaker2_source_connector] import ( "context" "fmt" "io" + managedkafka "cloud.google.com/go/managedkafka/apiv1" "cloud.google.com/go/managedkafka/apiv1/managedkafkapb" "google.golang.org/api/option" - - managedkafka "cloud.google.com/go/managedkafka/apiv1" ) -func createMirrorMakerConnector(w io.Writer, projectID, region, connectClusterID, connectorID, sourceDNS, targetDNS, topicName string, opts ...option.ClientOption) error { +// createMirrorMaker2SourceConnector creates a MirrorMaker 2.0 Source connector. +func createMirrorMaker2SourceConnector(w io.Writer, projectID, region, connectClusterID, connectorID, sourceBootstrapServers, targetBootstrapServers, tasksMax, sourceClusterAlias, targetClusterAlias, topics, topicsExclude string, opts ...option.ClientOption) error { + // TODO(developer): Update with your config values. Here is a sample configuration: // projectID := "my-project-id" // region := "us-central1" // connectClusterID := "my-connect-cluster" - // connectorID := "my-mm2-connector" - // sourceDNS := "source-cluster-dns:9092" - // targetDNS := "target-cluster-dns:9092" - // topicName := "my-topic" + // connectorID := "mm2-source-to-target-connector-id" + // sourceBootstrapServers := "source_cluster_dns" + // targetBootstrapServers := "target_cluster_dns" + // tasksMax := "3" + // sourceClusterAlias := "source" + // targetClusterAlias := "target" + // topics := ".*" + // topicsExclude := "mm2.*.internal,.*.replica,__.*" ctx := context.Background() client, err := managedkafka.NewManagedKafkaConnectClient(ctx, opts...) if err != nil { @@ -43,24 +48,22 @@ func createMirrorMakerConnector(w io.Writer, projectID, region, connectClusterID parent := fmt.Sprintf("projects/%s/locations/%s/connectClusters/%s", projectID, region, connectClusterID) - // MirrorMaker 2.0 sample connector configuration config := map[string]string{ - "connector.class": "org.apache.kafka.connect.mirror.MirrorSourceConnector", - "name": connectorID, - "source.cluster.alias": "source", - "target.cluster.alias": "target", - "topics": topicName, - "source.cluster.bootstrap.servers": sourceDNS, - "target.cluster.bootstrap.servers": targetDNS, - "offset-syncs.topic.replication.factor": "1", - "source.cluster.security.protocol": "SASL_SSL", - "source.cluster.sasl.mechanism": "OAUTHBEARER", - "source.cluster.sasl.login.callback.handler.class": "com.google.cloud.hosted.kafka.auth.GcpLoginCallbackHandler", - "source.cluster.sasl.jaas.config": "org.apache.kafka.common.security.oauthbearer.OAuthBearerLoginModule required;", - "target.cluster.security.protocol": "SASL_SSL", - "target.cluster.sasl.mechanism": "OAUTHBEARER", - "target.cluster.sasl.login.callback.handler.class": "com.google.cloud.hosted.kafka.auth.GcpLoginCallbackHandler", - "target.cluster.sasl.jaas.config": "org.apache.kafka.common.security.oauthbearer.OAuthBearerLoginModule required;", + "connector.class": "org.apache.kafka.connect.mirror.MirrorSourceConnector", + "name": connectorID, + "tasks.max": tasksMax, + "source.cluster.alias": sourceClusterAlias, + "target.cluster.alias": targetClusterAlias, // This is usually the primary cluster. + // Replicate all topics from the source + "topics": topics, + // The value for bootstrap.servers is a hostname:port pair for the Kafka broker in + // the source/target cluster. + // For example: "kafka-broker:9092" + "source.cluster.bootstrap.servers": sourceBootstrapServers, + "target.cluster.bootstrap.servers": targetBootstrapServers, + // You can define an exclusion policy for topics as follows: + // To exclude internal MirrorMaker 2 topics, internal topics and replicated topics. + "topics.exclude": topicsExclude, } connector := &managedkafkapb.Connector{ @@ -78,8 +81,8 @@ func createMirrorMakerConnector(w io.Writer, projectID, region, connectClusterID if err != nil { return fmt.Errorf("client.CreateConnector got err: %w", err) } - fmt.Fprintf(w, "Created MirrorMaker connector: %s\n", resp.Name) + fmt.Fprintf(w, "Created MirrorMaker 2.0 Source connector: %s\n", resp.Name) return nil } -// [END managedkafka_create_mirrormaker_connector] +// [END managedkafka_create_mirrormaker2_source_connector] \ No newline at end of file diff --git a/managedkafka/connect/connectors/create_pubsub_sink_connector.go b/managedkafka/connect/connectors/create_pubsub_sink_connector.go index 745e6243be..e915fbba32 100644 --- a/managedkafka/connect/connectors/create_pubsub_sink_connector.go +++ b/managedkafka/connect/connectors/create_pubsub_sink_connector.go @@ -6,7 +6,7 @@ // // https://www.apache.org/licenses/LICENSE-2.0 // -// Unless required by applicable law or agreed to in writing, software +// Unless required by applicable law of agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and @@ -20,19 +20,24 @@ import ( "fmt" "io" + managedkafka "cloud.google.com/go/managedkafka/apiv1" "cloud.google.com/go/managedkafka/apiv1/managedkafkapb" "google.golang.org/api/option" - - managedkafka "cloud.google.com/go/managedkafka/apiv1" ) -func createPubSubSinkConnector(w io.Writer, projectID, region, connectClusterID, connectorID, topicName, pubsubTopic string, opts ...option.ClientOption) error { +// createPubSubSinkConnector creates a Pub/Sub Sink connector. +func createPubSubSinkConnector(w io.Writer, projectID, region, connectClusterID, connectorID, topics, valueConverter, keyConverter, cpsTopic, cpsProject, tasksMax string, opts ...option.ClientOption) error { + // TODO(developer): Update with your config values. Here is a sample configuration: // projectID := "my-project-id" // region := "us-central1" // connectClusterID := "my-connect-cluster" - // connectorID := "my-pubsub-sink-connector" - // topicName := "my-kafka-topic" - // pubsubTopic := "my-pubsub-topic" + // connectorID := "CPS_SINK_CONNECTOR_ID" + // topics := "GMK_TOPIC_ID" + // valueConverter := "org.apache.kafka.connect.storage.StringConverter" + // keyConverter := "org.apache.kafka.connect.storage.StringConverter" + // cpsTopic := "CPS_TOPIC_ID" + // cpsProject := "GCP_PROJECT_ID" + // tasksMax := "3" ctx := context.Background() client, err := managedkafka.NewManagedKafkaConnectClient(ctx, opts...) if err != nil { @@ -46,12 +51,12 @@ func createPubSubSinkConnector(w io.Writer, projectID, region, connectClusterID, config := map[string]string{ "connector.class": "com.google.pubsub.kafka.sink.CloudPubSubSinkConnector", "name": connectorID, - "tasks.max": "1", - "topics": topicName, - "value.converter": "org.apache.kafka.connect.storage.StringConverter", - "key.converter": "org.apache.kafka.connect.storage.StringConverter", - "cps.topic": pubsubTopic, - "cps.project": projectID, + "tasks.max": tasksMax, + "topics": topics, + "value.converter": valueConverter, + "key.converter": keyConverter, + "cps.topic": cpsTopic, + "cps.project": cpsProject, } connector := &managedkafkapb.Connector{ diff --git a/managedkafka/connect/connectors/create_pubsub_source_connector.go b/managedkafka/connect/connectors/create_pubsub_source_connector.go index b3717fd7fc..e2f99975d4 100644 --- a/managedkafka/connect/connectors/create_pubsub_source_connector.go +++ b/managedkafka/connect/connectors/create_pubsub_source_connector.go @@ -20,19 +20,24 @@ import ( "fmt" "io" + managedkafka "cloud.google.com/go/managedkafka/apiv1" "cloud.google.com/go/managedkafka/apiv1/managedkafkapb" "google.golang.org/api/option" - - managedkafka "cloud.google.com/go/managedkafka/apiv1" ) -func createPubSubSourceConnector(w io.Writer, projectID, region, connectClusterID, connectorID, topicName, subscription string, opts ...option.ClientOption) error { +// createPubSubSourceConnector creates a Pub/Sub Source connector. +func createPubSubSourceConnector(w io.Writer, projectID, region, connectClusterID, connectorID, kafkaTopic, cpsSubscription, cpsProject, tasksMax, valueConverter, keyConverter string, opts ...option.ClientOption) error { + // TODO(developer): Update with your config values. Here is a sample configuration: // projectID := "my-project-id" // region := "us-central1" // connectClusterID := "my-connect-cluster" - // connectorID := "my-pubsub-source-connector" - // topicName := "my-kafka-topic" - // subscription := "my-pubsub-subscription" + // connectorID := "CPS_SOURCE_CONNECTOR_ID" + // kafkaTopic := "GMK_TOPIC_ID" + // cpsSubscription := "CPS_SUBSCRIPTION_ID" + // cpsProject := "GCP_PROJECT_ID" + // tasksMax := "3" + // valueConverter := "org.apache.kafka.connect.converters.ByteArrayConverter" + // keyConverter := "org.apache.kafka.connect.storage.StringConverter" ctx := context.Background() client, err := managedkafka.NewManagedKafkaConnectClient(ctx, opts...) if err != nil { @@ -46,12 +51,12 @@ func createPubSubSourceConnector(w io.Writer, projectID, region, connectClusterI config := map[string]string{ "connector.class": "com.google.pubsub.kafka.source.CloudPubSubSourceConnector", "name": connectorID, - "tasks.max": "1", - "kafka.topic": topicName, - "cps.subscription": subscription, - "cps.project": projectID, - "value.converter": "org.apache.kafka.connect.converters.ByteArrayConverter", - "key.converter": "org.apache.kafka.connect.storage.StringConverter", + "tasks.max": tasksMax, + "kafka.topic": kafkaTopic, + "cps.subscription": cpsSubscription, + "cps.project": cpsProject, + "value.converter": valueConverter, + "key.converter": keyConverter, } connector := &managedkafkapb.Connector{ @@ -74,3 +79,4 @@ func createPubSubSourceConnector(w io.Writer, projectID, region, connectClusterI } // [END managedkafka_create_pubsub_source_connector] + From 88d93e5876715d3b300692b76acea040fd8cbdf9 Mon Sep 17 00:00:00 2001 From: salmany Date: Tue, 9 Sep 2025 16:22:30 +0000 Subject: [PATCH 05/11] Add comment about accessible subnets and DNS domains. --- .../connect/clusters/create_connect_cluster.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/managedkafka/connect/clusters/create_connect_cluster.go b/managedkafka/connect/clusters/create_connect_cluster.go index 795096362a..61d5c3b861 100644 --- a/managedkafka/connect/clusters/create_connect_cluster.go +++ b/managedkafka/connect/clusters/create_connect_cluster.go @@ -47,6 +47,16 @@ func createConnectCluster(w io.Writer, projectID, region, clusterID, kafkaCluste MemoryBytes: 12884901888, // 12 GiB in bytes } + // Optionally, you can also specify accessible subnets and resolvable DNS + // domains as part of your network configuration. For example: + // networkConfigs := []*managedkafkapb.ConnectNetworkConfig{ + // { + // PrimarySubnet: primarySubnet, + // AdditionalSubnets: []string{"subnet-1", "subnet-2"}, + // DnsDomainNames: []string{"domain-1", "domain-2"}, + // }, + // } + connectCluster := &managedkafkapb.ConnectCluster{ Name: clusterPath, KafkaCluster: kafkaCluster, From eb1c642ea7d19cbe12b96251b2f280ad96b8a104 Mon Sep 17 00:00:00 2001 From: salmany Date: Tue, 9 Sep 2025 16:40:09 +0000 Subject: [PATCH 06/11] Fix memory defaults for connect cluster example. --- managedkafka/connect/clusters/connect_clusters_test.go | 2 +- managedkafka/connect/clusters/update_connect_cluster.go | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/managedkafka/connect/clusters/connect_clusters_test.go b/managedkafka/connect/clusters/connect_clusters_test.go index 7000276048..40fadc70b1 100644 --- a/managedkafka/connect/clusters/connect_clusters_test.go +++ b/managedkafka/connect/clusters/connect_clusters_test.go @@ -64,7 +64,7 @@ func TestConnectClusters(t *testing.T) { }) t.Run("UpdateConnectCluster", func(t *testing.T) { buf.Reset() - memoryBytes := int64(13958643712) // 13 GiB in bytes + memoryBytes := int64(25769803776) // 24 GiB in bytes labels := map[string]string{"environment": "test"} if err := updateConnectCluster(buf, tc.ProjectID, region, connectClusterID, memoryBytes, labels, options...); err != nil { t.Fatalf("failed to update connect cluster: %v", err) diff --git a/managedkafka/connect/clusters/update_connect_cluster.go b/managedkafka/connect/clusters/update_connect_cluster.go index 649a76812b..df4b07b609 100644 --- a/managedkafka/connect/clusters/update_connect_cluster.go +++ b/managedkafka/connect/clusters/update_connect_cluster.go @@ -31,7 +31,7 @@ func updateConnectCluster(w io.Writer, projectID, region, clusterID string, memo // projectID := "my-project-id" // region := "us-central1" // clusterID := "my-connect-cluster" - // memoryBytes := 13958643712 // 13 GiB in bytes + // memoryBytes := 25769803776 // 24 GiB in bytes // labels := map[string]string{"environment": "production"} ctx := context.Background() client, err := managedkafka.NewManagedKafkaConnectClient(ctx, opts...) @@ -41,12 +41,12 @@ func updateConnectCluster(w io.Writer, projectID, region, clusterID string, memo defer client.Close() clusterPath := fmt.Sprintf("projects/%s/locations/%s/connectClusters/%s", projectID, region, clusterID) - + // Capacity configuration update capacityConfig := &managedkafkapb.CapacityConfig{ MemoryBytes: memoryBytes, } - + connectCluster := &managedkafkapb.ConnectCluster{ Name: clusterPath, CapacityConfig: capacityConfig, From c1cec6f08ea6717b6a4f665a38945b827fc9bf44 Mon Sep 17 00:00:00 2001 From: salmany Date: Tue, 9 Sep 2025 16:47:30 +0000 Subject: [PATCH 07/11] Fix name of MM2 Source sample. --- ...nector.go => create_mirrormaker2_source_connector.go} | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) rename managedkafka/connect/connectors/{create_mirrormaker_connector.go => create_mirrormaker2_source_connector.go} (92%) diff --git a/managedkafka/connect/connectors/create_mirrormaker_connector.go b/managedkafka/connect/connectors/create_mirrormaker2_source_connector.go similarity index 92% rename from managedkafka/connect/connectors/create_mirrormaker_connector.go rename to managedkafka/connect/connectors/create_mirrormaker2_source_connector.go index b7cecaa9f8..2ff39b13cd 100644 --- a/managedkafka/connect/connectors/create_mirrormaker_connector.go +++ b/managedkafka/connect/connectors/create_mirrormaker2_source_connector.go @@ -49,9 +49,9 @@ func createMirrorMaker2SourceConnector(w io.Writer, projectID, region, connectCl parent := fmt.Sprintf("projects/%s/locations/%s/connectClusters/%s", projectID, region, connectClusterID) config := map[string]string{ - "connector.class": "org.apache.kafka.connect.mirror.MirrorSourceConnector", - "name": connectorID, - "tasks.max": tasksMax, + "connector.class": "org.apache.kafka.connect.mirror.MirrorSourceConnector", + "name": connectorID, + "tasks.max": tasksMax, "source.cluster.alias": sourceClusterAlias, "target.cluster.alias": targetClusterAlias, // This is usually the primary cluster. // Replicate all topics from the source @@ -63,6 +63,7 @@ func createMirrorMaker2SourceConnector(w io.Writer, projectID, region, connectCl "target.cluster.bootstrap.servers": targetBootstrapServers, // You can define an exclusion policy for topics as follows: // To exclude internal MirrorMaker 2 topics, internal topics and replicated topics. + // topicsExclude := "mm2.*.internal,.*.replica,__.*" "topics.exclude": topicsExclude, } @@ -85,4 +86,4 @@ func createMirrorMaker2SourceConnector(w io.Writer, projectID, region, connectCl return nil } -// [END managedkafka_create_mirrormaker2_source_connector] \ No newline at end of file +// [END managedkafka_create_mirrormaker2_source_connector] From 280815fe214f66127acee12969a868bd311cb7f4 Mon Sep 17 00:00:00 2001 From: salmany Date: Wed, 10 Sep 2025 16:15:31 +0000 Subject: [PATCH 08/11] Fix sample parameter for Cloud Storage Sink connector. --- managedkafka/connect/connectors/connectors_test.go | 3 ++- .../connectors/create_cloud_storage_sink_connector.go | 5 +++-- managedkafka/connect/connectors/update_connector.go | 2 +- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/managedkafka/connect/connectors/connectors_test.go b/managedkafka/connect/connectors/connectors_test.go index 9699225f79..740f0095bd 100644 --- a/managedkafka/connect/connectors/connectors_test.go +++ b/managedkafka/connect/connectors/connectors_test.go @@ -116,8 +116,9 @@ func TestConnectors(t *testing.T) { keyConverter := "org.apache.kafka.connect.storage.StringConverter" valueConverter := "org.apache.kafka.connect.json.JsonConverter" valueConverterSchemasEnable := "false" + gcsCredentialsDefault := "true" defaultDataset := "test-dataset" - if err := createBigQuerySinkConnector(buf, tc.ProjectID, region, connectClusterID, connectorID+"-bq-sink", topics, tasksMax, keyConverter, valueConverter, valueConverterSchemasEnable, defaultDataset, options...); err != nil { + if err := createBigQuerySinkConnector(buf, tc.ProjectID, region, connectClusterID, connectorID+"-bq-sink", topics, tasksMax, keyConverter, gcsCredentialsDefault, valueConverter, valueConverterSchemasEnable, defaultDataset, options...); err != nil { t.Fatalf("failed to create BigQuery sink connector: %v", err) } got := buf.String() diff --git a/managedkafka/connect/connectors/create_cloud_storage_sink_connector.go b/managedkafka/connect/connectors/create_cloud_storage_sink_connector.go index ab2a42ba60..4d01c67223 100644 --- a/managedkafka/connect/connectors/create_cloud_storage_sink_connector.go +++ b/managedkafka/connect/connectors/create_cloud_storage_sink_connector.go @@ -26,7 +26,7 @@ import ( ) // createCloudStorageSinkConnector creates a Cloud Storage Sink connector. -func createCloudStorageSinkConnector(w io.Writer, projectID, region, connectClusterID, connectorID, topics, gcsBucketName, tasksMax, formatOutputType, valueConverter, valueConverterSchemasEnable, keyConverter string, opts ...option.ClientOption) error { +func createCloudStorageSinkConnector(w io.Writer, projectID, region, connectClusterID, connectorID, topics, gcsBucketName, tasksMax, formatOutputType, valueConverter, valueConverterSchemasEnable, keyConverter, gcsCredentialsDefault string, opts ...option.ClientOption) error { // TODO(developer): Update with your config values. Here is a sample configuration: // projectID := "my-project-id" // region := "us-central1" @@ -39,6 +39,7 @@ func createCloudStorageSinkConnector(w io.Writer, projectID, region, connectClus // valueConverter := "org.apache.kafka.connect.json.JsonConverter" // valueConverterSchemasEnable := "false" // keyConverter := "org.apache.kafka.connect.storage.StringConverter" + // gcsCredentialsDefault := "true" ctx := context.Background() client, err := managedkafka.NewManagedKafkaConnectClient(ctx, opts...) if err != nil { @@ -53,7 +54,7 @@ func createCloudStorageSinkConnector(w io.Writer, projectID, region, connectClus "tasks.max": tasksMax, "topics": topics, "gcs.bucket.name": gcsBucketName, - "gcs.credentials.default": "true", + "gcs.credentials.default": gcsCredentialsDefault, "format.output.type": formatOutputType, "name": connectorID, "value.converter": valueConverter, diff --git a/managedkafka/connect/connectors/update_connector.go b/managedkafka/connect/connectors/update_connector.go index 6300c5b12c..57c1f3a580 100644 --- a/managedkafka/connect/connectors/update_connector.go +++ b/managedkafka/connect/connectors/update_connector.go @@ -32,7 +32,7 @@ func updateConnector(w io.Writer, projectID, region, connectClusterID, connector // region := "us-central1" // connectClusterID := "my-connect-cluster" // connectorID := "my-connector" - // config := map[string]string{"tasks.max": "2"} + // config := map[string]string{"tasks.max": "6"} ctx := context.Background() client, err := managedkafka.NewManagedKafkaConnectClient(ctx, opts...) if err != nil { From f5629078d177b6877917952d7d95f8691a276b4b Mon Sep 17 00:00:00 2001 From: salmany Date: Fri, 12 Sep 2025 01:47:55 +0000 Subject: [PATCH 09/11] Fix linting errors. --- internal/managedkafka/fake/fake.go | 10 +- .../create_pubsub_source_connector.go | 1 - managedkafka/go.sum | 174 +++++++++--------- 3 files changed, 93 insertions(+), 92 deletions(-) diff --git a/internal/managedkafka/fake/fake.go b/internal/managedkafka/fake/fake.go index 0488e39dfa..8bbde11a97 100644 --- a/internal/managedkafka/fake/fake.go +++ b/internal/managedkafka/fake/fake.go @@ -30,11 +30,11 @@ import ( ) const ( - clusterID = "fake-cluster" - topicID = "fake-topic" - consumerGroupID = "fake-consumergroup" - connectClusterID = "fake-connect-cluster" - connectorID = "fake-connector" + clusterID = "fake-cluster" + topicID = "fake-topic" + consumerGroupID = "fake-consumergroup" + connectClusterID = "fake-connect-cluster" + connectorID = "fake-connector" ) // The reason why we have a fake server is because testing end-to-end will exceed the deadline of 10 minutes. diff --git a/managedkafka/connect/connectors/create_pubsub_source_connector.go b/managedkafka/connect/connectors/create_pubsub_source_connector.go index e2f99975d4..d98dfc6e4a 100644 --- a/managedkafka/connect/connectors/create_pubsub_source_connector.go +++ b/managedkafka/connect/connectors/create_pubsub_source_connector.go @@ -79,4 +79,3 @@ func createPubSubSourceConnector(w io.Writer, projectID, region, connectClusterI } // [END managedkafka_create_pubsub_source_connector] - diff --git a/managedkafka/go.sum b/managedkafka/go.sum index 71667acab8..76269f27c2 100644 --- a/managedkafka/go.sum +++ b/managedkafka/go.sum @@ -1,127 +1,129 @@ -cel.dev/expr v0.19.1 h1:NciYrtDRIR0lNCnH1LFJegdjspNx9fI59O7TWcua/W4= -cel.dev/expr v0.19.1/go.mod h1:MrpN08Q+lEBs+bGYdLxxHkZoUSsCp0nSKTs0nTymJgw= -cloud.google.com/go v0.118.0 h1:tvZe1mgqRxpiVa3XlIGMiPcEUbP1gNXELgD4y/IXmeQ= -cloud.google.com/go v0.118.0/go.mod h1:zIt2pkedt/mo+DQjcT4/L3NDxzHPR29j5HcclNH+9PM= -cloud.google.com/go/auth v0.14.0 h1:A5C4dKV/Spdvxcl0ggWwWEzzP7AZMJSEIgrkngwhGYM= -cloud.google.com/go/auth v0.14.0/go.mod h1:CYsoRL1PdiDuqeQpZE0bP2pnPrGqFcOkI0nldEQis+A= -cloud.google.com/go/auth/oauth2adapt v0.2.7 h1:/Lc7xODdqcEw8IrZ9SvwnlLX6j9FHQM74z6cBk9Rw6M= -cloud.google.com/go/auth/oauth2adapt v0.2.7/go.mod h1:NTbTTzfvPl1Y3V1nPpOgl2w6d/FjO7NNUQaWSox6ZMc= -cloud.google.com/go/compute/metadata v0.6.0 h1:A6hENjEsCDtC1k8byVsgwvVcioamEHvZ4j01OwKxG9I= -cloud.google.com/go/compute/metadata v0.6.0/go.mod h1:FjyFAW1MW0C203CEOMDTu3Dk1FlqW3Rga40jzHL4hfg= -cloud.google.com/go/iam v1.3.1 h1:KFf8SaT71yYq+sQtRISn90Gyhyf4X8RGgeAVC8XGf3E= -cloud.google.com/go/iam v1.3.1/go.mod h1:3wMtuyT4NcbnYNPLMBzYRFiEfjKfJlLVLrisE7bwm34= +cel.dev/expr v0.23.0 h1:wUb94w6OYQS4uXraxo9U+wUAs9jT47Xvl4iPgAwM2ss= +cel.dev/expr v0.23.0/go.mod h1:hLPLo1W4QUmuYdA72RBX06QTs6MXw941piREPl3Yfiw= +cloud.google.com/go v0.120.0 h1:wc6bgG9DHyKqF5/vQvX1CiZrtHnxJjBlKUyF9nP6meA= +cloud.google.com/go v0.120.0/go.mod h1:/beW32s8/pGRuj4IILWQNd4uuebeT4dkOhKmkfit64Q= +cloud.google.com/go/auth v0.16.2 h1:QvBAGFPLrDeoiNjyfVunhQ10HKNYuOwZ5noee0M5df4= +cloud.google.com/go/auth v0.16.2/go.mod h1:sRBas2Y1fB1vZTdurouM0AzuYQBMZinrUYL8EufhtEA= +cloud.google.com/go/auth/oauth2adapt v0.2.8 h1:keo8NaayQZ6wimpNSmW5OPc283g65QNIiLpZnkHRbnc= +cloud.google.com/go/auth/oauth2adapt v0.2.8/go.mod h1:XQ9y31RkqZCcwJWNSx2Xvric3RrU88hAYYbjDWYDL+c= +cloud.google.com/go/compute/metadata v0.7.0 h1:PBWF+iiAerVNe8UCHxdOt6eHLVc3ydFeOCw78U8ytSU= +cloud.google.com/go/compute/metadata v0.7.0/go.mod h1:j5MvL9PprKL39t166CoB1uVHfQMs4tFQZZcKwksXUjo= +cloud.google.com/go/iam v1.5.2 h1:qgFRAGEmd8z6dJ/qyEchAuL9jpswyODjA2lS+w234g8= +cloud.google.com/go/iam v1.5.2/go.mod h1:SE1vg0N81zQqLzQEwxL2WI6yhetBdbNQuTvIKCSkUHE= cloud.google.com/go/logging v1.13.0 h1:7j0HgAp0B94o1YRDqiqm26w4q1rDMH7XNRU34lJXHYc= cloud.google.com/go/logging v1.13.0/go.mod h1:36CoKh6KA/M0PbhPKMq6/qety2DCAErbhXT62TuXALA= -cloud.google.com/go/longrunning v0.6.4 h1:3tyw9rO3E2XVXzSApn1gyEEnH2K9SynNQjMlBi3uHLg= -cloud.google.com/go/longrunning v0.6.4/go.mod h1:ttZpLCe6e7EXvn9OxpBRx7kZEB0efv8yBO6YnVMfhJs= -cloud.google.com/go/managedkafka v0.1.3 h1:sctnVM1h86FaI+BWPbgSccgBb36WFDKpu6qRvfmUSzU= -cloud.google.com/go/managedkafka v0.1.3/go.mod h1:N9i335Os/rPILFKQfKvVbAQtLblPYFco0nFl4/3G6lk= -cloud.google.com/go/monitoring v1.23.0 h1:M3nXww2gn9oZ/qWN2bZ35CjolnVHM3qnSbu6srCPgjk= -cloud.google.com/go/monitoring v1.23.0/go.mod h1:034NnlQPDzrQ64G2Gavhl0LUHZs9H3rRmhtnp7jiJgg= +cloud.google.com/go/longrunning v0.6.7 h1:IGtfDWHhQCgCjwQjV9iiLnUta9LBCo8R9QmAFsS/PrE= +cloud.google.com/go/longrunning v0.6.7/go.mod h1:EAFV3IZAKmM56TyiE6VAP3VoTzhZzySwI/YI1s/nRsY= +cloud.google.com/go/managedkafka v0.7.0 h1:iq8a6hQiZN31lxclaEItx8znhsDmG8b4aOGuFRM2mwE= +cloud.google.com/go/managedkafka v0.7.0/go.mod h1:mYuHJhUaC9m0n8fbdAMdYfG9e242/13R3GMUz/dDRsM= +cloud.google.com/go/monitoring v1.24.2 h1:5OTsoJ1dXYIiMiuL+sYscLc9BumrL3CarVLL7dd7lHM= +cloud.google.com/go/monitoring v1.24.2/go.mod h1:x7yzPWcgDRnPEv3sI+jJGBkwl5qINf+6qY4eq0I9B4U= cloud.google.com/go/storage v1.50.0 h1:3TbVkzTooBvnZsk7WaAQfOsNrdoM8QHusXA1cpk6QJs= cloud.google.com/go/storage v1.50.0/go.mod h1:l7XeiD//vx5lfqE3RavfmU9yvk5Pp0Zhcv482poyafY= -cloud.google.com/go/trace v1.11.3 h1:c+I4YFjxRQjvAhRmSsmjpASUKq88chOX854ied0K/pE= -cloud.google.com/go/trace v1.11.3/go.mod h1:pt7zCYiDSQjC9Y2oqCsh9jF4GStB/hmjrYLsxRR27q8= +cloud.google.com/go/trace v1.11.6 h1:2O2zjPzqPYAHrn3OKl029qlqG6W8ZdYaOWRyr8NgMT4= +cloud.google.com/go/trace v1.11.6/go.mod h1:GA855OeDEBiBMzcckLPE2kDunIpC72N+Pq8WFieFjnI= github.com/GoogleCloudPlatform/golang-samples v0.0.0-20240724083556-7f760db013b7 h1:yGCaiv5IE3WoRTUOXHD/jybC2RIGTdCKuNwwmwQq7u4= github.com/GoogleCloudPlatform/golang-samples v0.0.0-20240724083556-7f760db013b7/go.mod h1:CK/v6fB0p6JTQtDAQ1UyKABPBiHRsA3+qbX2yuZZk1w= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.25.0 h1:3c8yed4lgqTt+oTQ+JNMDo+F4xprBf+O/il4ZC0nRLw= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.25.0/go.mod h1:obipzmGjfSjam60XLwGfqUkJsfiheAl+TUjG+4yzyPM= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.49.0 h1:o90wcURuxekmXrtxmYWTyNla0+ZEHhud6DI1ZTxd1vI= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.49.0/go.mod h1:6fTWu4m3jocfUZLYF5KsZC1TUfRvEjs7lM4crme/irw= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.49.0 h1:jJKWl98inONJAr/IZrdFQUWcwUO95DLY1XMD1ZIut+g= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.49.0/go.mod h1:l2fIqmwB+FKSfvn3bAD/0i+AXAxhIZjTK2svT/mgUXs= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.49.0 h1:GYUJLfvd++4DMuMhCFLgLXvFwofIxh/qOwoGuS/LTew= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.49.0/go.mod h1:wRbFgBQUVm1YXrvWKofAEmq9HNJTDphbAaJSSX01KUI= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.27.0 h1:ErKg/3iS1AKcTkf3yixlZ54f9U1rljCkQyEXWUnIUxc= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.27.0/go.mod h1:yAZHSGnqScoU556rBOVkwLze6WP5N+U11RHuWaGVxwY= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.50.0 h1:5IT7xOdq17MtcdtL/vtl6mGfzhaq4m4vpollPRmlsBQ= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.50.0/go.mod h1:ZV4VOm0/eHR06JLrXWe09068dHpr3TRpY9Uo7T+anuA= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.50.0 h1:nNMpRpnkWDAaqcpxMJvxa/Ud98gjbYwayJY4/9bdjiU= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.50.0/go.mod h1:SZiPHWGOOk3bl8tkevxkoiwPgsIl6CwrWcbwjfHZpdM= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.50.0 h1:ig/FpDD2JofP/NExKQUbn7uOSZzJAQqogfqluZK4ed4= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.50.0/go.mod h1:otE2jQekW/PqXk1Awf5lmfokJx4uwuqcj1ab5SpGeW0= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/cncf/xds/go v0.0.0-20250121191232-2f005788dc42 h1:Om6kYQYDUk5wWbT0t0q6pvyM49i9XZAv9dDrkDA7gjk= -github.com/cncf/xds/go v0.0.0-20250121191232-2f005788dc42/go.mod h1:W+zGtBO5Y1IgJhy4+A9GOqVhqLpfZi+vwmdNXUehLA8= +github.com/cncf/xds/go v0.0.0-20250326154945-ae57f3c0d45f h1:C5bqEmzEPLsHm9Mv73lSE9e9bKV23aB1vxOsmZrkl3k= +github.com/cncf/xds/go v0.0.0-20250326154945-ae57f3c0d45f/go.mod h1:W+zGtBO5Y1IgJhy4+A9GOqVhqLpfZi+vwmdNXUehLA8= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/envoyproxy/go-control-plane v0.13.4 h1:zEqyPVyku6IvWCFwux4x9RxkLOMUL+1vC9xUFv5l2/M= github.com/envoyproxy/go-control-plane v0.13.4/go.mod h1:kDfuBlDVsSj2MjrLEtRWtHlsWIFcGyB2RMO44Dc5GZA= -github.com/envoyproxy/go-control-plane/envoy v1.32.3 h1:hVEaommgvzTjTd4xCaFd+kEQ2iYBtGxP6luyLrx6uOk= -github.com/envoyproxy/go-control-plane/envoy v1.32.3/go.mod h1:F6hWupPfh75TBXGKA++MCT/CZHFq5r9/uwt/kQYkZfE= +github.com/envoyproxy/go-control-plane/envoy v1.32.4 h1:jb83lalDRZSpPWW2Z7Mck/8kXZ5CQAFYVjQcdVIr83A= +github.com/envoyproxy/go-control-plane/envoy v1.32.4/go.mod h1:Gzjc5k8JcJswLjAx1Zm+wSYE20UrLtt7JZMWiWQXQEw= github.com/envoyproxy/go-control-plane/ratelimit v0.1.0 h1:/G9QYbddjL25KvtKTv3an9lx6VBE2cnb8wp1vEGNYGI= github.com/envoyproxy/go-control-plane/ratelimit v0.1.0/go.mod h1:Wk+tMFAFbCXaJPzVVHnPgRKdUdwW/KdbRt94AzgRee4= -github.com/envoyproxy/protoc-gen-validate v1.1.0 h1:tntQDh69XqOCOZsDz0lVJQez/2L6Uu2PdjCQwWCJ3bM= -github.com/envoyproxy/protoc-gen-validate v1.1.0/go.mod h1:sXRDRVmzEbkM7CVcM06s9shE/m23dg3wzjl0UWqJ2q4= +github.com/envoyproxy/protoc-gen-validate v1.2.1 h1:DEo3O99U8j4hBFwbJfrz9VtgcDfUKS7KJ7spH3d86P8= +github.com/envoyproxy/protoc-gen-validate v1.2.1/go.mod h1:d/C80l/jxXLdfEIhX1W2TmLfsJ31lvEjwamM4DxlWXU= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/go-jose/go-jose/v4 v4.0.5 h1:M6T8+mKZl/+fNNuFHvGIzDz7BTLQPIounk/b9dw3AaE= +github.com/go-jose/go-jose/v4 v4.0.5/go.mod h1:s3P1lRrkT8igV8D9OjyL4WRyHvjB6a4JSllnOrmmBOA= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= -github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 h1:f+oWsMOmNPc8JmEHVZIycC7hBoQxHH9pNKQORJNozsQ= -github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8/go.mod h1:wcDNUvekVysuuOpQKo3191zZyTpiI6se1N1ULghS0sw= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= -github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= -github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/martian/v3 v3.3.3 h1:DIhPTQrbPkgs2yJYdXU/eNACCG5DVQjySNRNlflZ9Fc= github.com/google/martian/v3 v3.3.3/go.mod h1:iEPrYcgCF7jA9OtScMFQyAlZZ4YXTKEtJ1E6RWzmBA0= github.com/google/s2a-go v0.1.9 h1:LGD7gtMgezd8a/Xak7mEWL0PjoTQFvpRudN895yqKW0= github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0dXayM= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/googleapis/enterprise-certificate-proxy v0.3.4 h1:XYIDZApgAnrN1c855gTgghdIA6Stxb52D5RnLI1SLyw= -github.com/googleapis/enterprise-certificate-proxy v0.3.4/go.mod h1:YKe7cfqYXjKGpGvmSg28/fFvhNzinZQm8DGnaburhGA= -github.com/googleapis/gax-go/v2 v2.14.1 h1:hb0FFeiPaQskmvakKu5EbCbpntQn48jyHuvrkurSS/Q= -github.com/googleapis/gax-go/v2 v2.14.1/go.mod h1:Hb/NubMaVM88SrNkvl8X/o8XWwDJEPqouaLeN2IUxoA= +github.com/googleapis/enterprise-certificate-proxy v0.3.6 h1:GW/XbdyBFQ8Qe+YAmFU9uHLo7OnF5tL52HFAgMmyrf4= +github.com/googleapis/enterprise-certificate-proxy v0.3.6/go.mod h1:MkHOF77EYAE7qfSuSS9PU6g4Nt4e11cnsDUowfwewLA= +github.com/googleapis/gax-go/v2 v2.14.2 h1:eBLnkZ9635krYIPD+ag1USrOAI0Nr0QYF3+/3GqO0k0= +github.com/googleapis/gax-go/v2 v2.14.2/go.mod h1:ON64QhlJkhVtSqp4v1uaK92VyZ2gmvDQsweuyLV+8+w= github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo= github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/spiffe/go-spiffe/v2 v2.5.0 h1:N2I01KCUkv1FAjZXJMwh95KK1ZIQLYbPfhaxw8WS0hE= +github.com/spiffe/go-spiffe/v2 v2.5.0/go.mod h1:P+NxobPc6wXhVtINNtFjNWGBTreew1GBUCwT2wPmb7g= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= -go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= +github.com/zeebo/errs v1.4.0 h1:XNdoD/RRMKP7HD0UhJnIzUy74ISdGGxURlYG8HSWSfM= +github.com/zeebo/errs v1.4.0/go.mod h1:sgbWHsvVuTPHcqJJGQ1WhI5KbWlHYz+2+2C/LSEtCw4= go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= -go.opentelemetry.io/contrib/detectors/gcp v1.34.0 h1:JRxssobiPg23otYU5SbWtQC//snGVIM3Tx6QRzlQBao= -go.opentelemetry.io/contrib/detectors/gcp v1.34.0/go.mod h1:cV4BMFcscUR/ckqLkbfQmF0PRsq8w/lMGzdbCSveBHo= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.59.0 h1:rgMkmiGfix9vFJDcDi1PK8WEQP4FLQwLDfhp5ZLpFeE= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.59.0/go.mod h1:ijPqXp5P6IRRByFVVg9DY8P5HkxkHE5ARIa+86aXPf4= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.59.0 h1:CV7UdSGJt/Ao6Gp4CXckLxVRRsRgDHoI8XjbL3PDl8s= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.59.0/go.mod h1:FRmFuRJfag1IZ2dPkHnEoSFVgTVPUd2qf5Vi69hLb8I= -go.opentelemetry.io/otel v1.34.0 h1:zRLXxLCgL1WyKsPVrgbSdMN4c0FMkDAskSTQP+0hdUY= -go.opentelemetry.io/otel v1.34.0/go.mod h1:OWFPOQ+h4G8xpyjgqo4SxJYdDQ/qmRH+wivy7zzx9oI= +go.opentelemetry.io/contrib/detectors/gcp v1.35.0 h1:bGvFt68+KTiAKFlacHW6AhA56GF2rS0bdD3aJYEnmzA= +go.opentelemetry.io/contrib/detectors/gcp v1.35.0/go.mod h1:qGWP8/+ILwMRIUf9uIVLloR1uo5ZYAslM4O6OqUi1DA= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.61.0 h1:q4XOmH/0opmeuJtPsbFNivyl7bCt7yRBbeEm2sC/XtQ= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.61.0/go.mod h1:snMWehoOh2wsEwnvvwtDyFCxVeDAODenXHtn5vzrKjo= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 h1:F7Jx+6hwnZ41NSFTO5q4LYDtJRXBf2PD0rNBkeB/lus= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0/go.mod h1:UHB22Z8QsdRDrnAtX4PntOl36ajSxcdUMt1sF7Y6E7Q= +go.opentelemetry.io/otel v1.36.0 h1:UumtzIklRBY6cI/lllNZlALOF5nNIzJVb16APdvgTXg= +go.opentelemetry.io/otel v1.36.0/go.mod h1:/TcFMXYjyRNh8khOAO9ybYkqaDBb/70aVwkNML4pP8E= go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.29.0 h1:WDdP9acbMYjbKIyJUhTvtzj601sVJOqgWdUxSdR/Ysc= go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.29.0/go.mod h1:BLbf7zbNIONBLPwvFnwNHGj4zge8uTCM/UPIVW1Mq2I= -go.opentelemetry.io/otel/metric v1.34.0 h1:+eTR3U0MyfWjRDhmFMxe2SsW64QrZ84AOhvqS7Y+PoQ= -go.opentelemetry.io/otel/metric v1.34.0/go.mod h1:CEDrp0fy2D0MvkXE+dPV7cMi8tWZwX3dmaIhwPOaqHE= -go.opentelemetry.io/otel/sdk v1.34.0 h1:95zS4k/2GOy069d321O8jWgYsW3MzVV+KuSPKp7Wr1A= -go.opentelemetry.io/otel/sdk v1.34.0/go.mod h1:0e/pNiaMAqaykJGKbi+tSjWfNNHMTxoC9qANsCzbyxU= -go.opentelemetry.io/otel/sdk/metric v1.34.0 h1:5CeK9ujjbFVL5c1PhLuStg1wxA7vQv7ce1EK0Gyvahk= -go.opentelemetry.io/otel/sdk/metric v1.34.0/go.mod h1:jQ/r8Ze28zRKoNRdkjCZxfs6YvBTG1+YIqyFVFYec5w= -go.opentelemetry.io/otel/trace v1.34.0 h1:+ouXS2V8Rd4hp4580a8q23bg0azF2nI8cqLYnC8mh/k= -go.opentelemetry.io/otel/trace v1.34.0/go.mod h1:Svm7lSjQD7kG7KJ/MUHPVXSDGz2OX4h0M2jHBhmSfRE= -golang.org/x/crypto v0.32.0 h1:euUpcYgM8WcP71gNpTqQCn6rC2t6ULUPiOzfWaXVVfc= -golang.org/x/crypto v0.32.0/go.mod h1:ZnnJkOaASj8g0AjIduWNlq2NRxL0PlBrbKVyZ6V/Ugc= -golang.org/x/net v0.34.0 h1:Mb7Mrk043xzHgnRM88suvJFwzVrRfHEHJEl5/71CKw0= -golang.org/x/net v0.34.0/go.mod h1:di0qlW3YNM5oh6GqDGQr92MyTozJPmybPK4Ev/Gm31k= -golang.org/x/oauth2 v0.25.0 h1:CY4y7XT9v0cRI9oupztF8AgiIu99L/ksR/Xp/6jrZ70= -golang.org/x/oauth2 v0.25.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= -golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= -golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= -golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU= -golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= -golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= -golang.org/x/time v0.9.0 h1:EsRrnYcQiGH+5FfbgvV4AP7qEZstoyrHB0DzarOQ4ZY= -golang.org/x/time v0.9.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= -google.golang.org/api v0.217.0 h1:GYrUtD289o4zl1AhiTZL0jvQGa2RDLyC+kX1N/lfGOU= -google.golang.org/api v0.217.0/go.mod h1:qMc2E8cBAbQlRypBTBWHklNJlaZZJBwDv81B1Iu8oSI= -google.golang.org/genproto v0.0.0-20250115164207-1a7da9e5054f h1:387Y+JbxF52bmesc8kq1NyYIp33dnxCw6eiA7JMsTmw= -google.golang.org/genproto v0.0.0-20250115164207-1a7da9e5054f/go.mod h1:0joYwWwLQh18AOj8zMYeZLjzuqcYTU3/nC5JdCvC3JI= -google.golang.org/genproto/googleapis/api v0.0.0-20250115164207-1a7da9e5054f h1:gap6+3Gk41EItBuyi4XX/bp4oqJ3UwuIMl25yGinuAA= -google.golang.org/genproto/googleapis/api v0.0.0-20250115164207-1a7da9e5054f/go.mod h1:Ic02D47M+zbarjYYUlK57y316f2MoN0gjAwI3f2S95o= -google.golang.org/genproto/googleapis/rpc v0.0.0-20250115164207-1a7da9e5054f h1:OxYkA3wjPsZyBylwymxSHa7ViiW1Sml4ToBrncvFehI= -google.golang.org/genproto/googleapis/rpc v0.0.0-20250115164207-1a7da9e5054f/go.mod h1:+2Yz8+CLJbIfL9z73EW45avw8Lmge3xVElCP9zEKi50= -google.golang.org/grpc v1.69.4 h1:MF5TftSMkd8GLw/m0KM6V8CMOCY6NZ1NQDPGFgbTt4A= -google.golang.org/grpc v1.69.4/go.mod h1:vyjdE6jLBI76dgpDojsFGNaHlxdjXN9ghpnd2o7JGZ4= -google.golang.org/protobuf v1.36.3 h1:82DV7MYdb8anAVi3qge1wSnMDrnKK7ebr+I0hHRN1BU= -google.golang.org/protobuf v1.36.3/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= +go.opentelemetry.io/otel/metric v1.36.0 h1:MoWPKVhQvJ+eeXWHFBOPoBOi20jh6Iq2CcCREuTYufE= +go.opentelemetry.io/otel/metric v1.36.0/go.mod h1:zC7Ks+yeyJt4xig9DEw9kuUFe5C3zLbVjV2PzT6qzbs= +go.opentelemetry.io/otel/sdk v1.36.0 h1:b6SYIuLRs88ztox4EyrvRti80uXIFy+Sqzoh9kFULbs= +go.opentelemetry.io/otel/sdk v1.36.0/go.mod h1:+lC+mTgD+MUWfjJubi2vvXWcVxyr9rmlshZni72pXeY= +go.opentelemetry.io/otel/sdk/metric v1.36.0 h1:r0ntwwGosWGaa0CrSt8cuNuTcccMXERFwHX4dThiPis= +go.opentelemetry.io/otel/sdk/metric v1.36.0/go.mod h1:qTNOhFDfKRwX0yXOqJYegL5WRaW376QbB7P4Pb0qva4= +go.opentelemetry.io/otel/trace v1.36.0 h1:ahxWNuqZjpdiFAyrIoQ4GIiAIhxAunQR6MUoKrsNd4w= +go.opentelemetry.io/otel/trace v1.36.0/go.mod h1:gQ+OnDZzrybY4k4seLzPAWNwVBBVlF2szhehOBB/tGA= +golang.org/x/crypto v0.39.0 h1:SHs+kF4LP+f+p14esP5jAoDpHU8Gu/v9lFRK6IT5imM= +golang.org/x/crypto v0.39.0/go.mod h1:L+Xg3Wf6HoL4Bn4238Z6ft6KfEpN0tJGo53AAPC632U= +golang.org/x/net v0.41.0 h1:vBTly1HeNPEn3wtREYfy4GZ/NECgw2Cnl+nK6Nz3uvw= +golang.org/x/net v0.41.0/go.mod h1:B/K4NNqkfmg07DQYrbwvSluqCJOOXwUjeb/5lOisjbA= +golang.org/x/oauth2 v0.30.0 h1:dnDm7JmhM45NNpd8FDDeLhK6FwqbOf4MLCM9zb1BOHI= +golang.org/x/oauth2 v0.30.0/go.mod h1:B++QgG3ZKulg6sRPGD/mqlHQs5rB3Ml9erfeDY7xKlU= +golang.org/x/sync v0.15.0 h1:KWH3jNZsfyT6xfAfKiz6MRNmd46ByHDYaZ7KSkCtdW8= +golang.org/x/sync v0.15.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw= +golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/text v0.26.0 h1:P42AVeLghgTYr4+xUnTRKDMqpar+PtX7KWuNQL21L8M= +golang.org/x/text v0.26.0/go.mod h1:QK15LZJUUQVJxhz7wXgxSy/CJaTFjd0G+YLonydOVQA= +golang.org/x/time v0.12.0 h1:ScB/8o8olJvc+CQPWrK3fPZNfh7qgwCrY0zJmoEQLSE= +golang.org/x/time v0.12.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= +google.golang.org/api v0.239.0 h1:2hZKUnFZEy81eugPs4e2XzIJ5SOwQg0G82bpXD65Puo= +google.golang.org/api v0.239.0/go.mod h1:cOVEm2TpdAGHL2z+UwyS+kmlGr3bVWQQ6sYEqkKje50= +google.golang.org/genproto v0.0.0-20250505200425-f936aa4a68b2 h1:1tXaIXCracvtsRxSBsYDiSBN0cuJvM7QYW+MrpIRY78= +google.golang.org/genproto v0.0.0-20250505200425-f936aa4a68b2/go.mod h1:49MsLSx0oWMOZqcpB3uL8ZOkAh1+TndpJ8ONoCBWiZk= +google.golang.org/genproto/googleapis/api v0.0.0-20250603155806-513f23925822 h1:oWVWY3NzT7KJppx2UKhKmzPq4SRe0LdCijVRwvGeikY= +google.golang.org/genproto/googleapis/api v0.0.0-20250603155806-513f23925822/go.mod h1:h3c4v36UTKzUiuaOKQ6gr3S+0hovBtUrXzTG/i3+XEc= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250603155806-513f23925822 h1:fc6jSaCT0vBduLYZHYrBBNY4dsWuvgyff9noRNDdBeE= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250603155806-513f23925822/go.mod h1:qQ0YXyHHx3XkvlzUtpXDkS29lDSafHMZBAZDc03LQ3A= +google.golang.org/grpc v1.73.0 h1:VIWSmpI2MegBtTuFt5/JWy2oXxtjJ/e89Z70ImfD2ok= +google.golang.org/grpc v1.73.0/go.mod h1:50sbHOUqWoCQGI8V2HQLJM0B+LMlIUjNSZmow7EVBQc= +google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY= +google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= From df7d66529e0fbcbbda1f757368e7c00d0f82479d Mon Sep 17 00:00:00 2001 From: salmany Date: Fri, 12 Sep 2025 01:50:48 +0000 Subject: [PATCH 10/11] Fix test error. --- managedkafka/connect/connectors/connectors_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/managedkafka/connect/connectors/connectors_test.go b/managedkafka/connect/connectors/connectors_test.go index 740f0095bd..d4d8cb20c1 100644 --- a/managedkafka/connect/connectors/connectors_test.go +++ b/managedkafka/connect/connectors/connectors_test.go @@ -99,7 +99,8 @@ func TestConnectors(t *testing.T) { valueConverter := "org.apache.kafka.connect.json.JsonConverter" valueConverterSchemasEnable := "false" keyConverter := "org.apache.kafka.connect.storage.StringConverter" - if err := createCloudStorageSinkConnector(buf, tc.ProjectID, region, connectClusterID, connectorID+"-gcs-sink", topics, gcsBucketName, tasksMax, formatOutputType, valueConverter, valueConverterSchemasEnable, keyConverter, options...); err != nil { + gcsCredentialsDefault := "true" + if err := createCloudStorageSinkConnector(buf, tc.ProjectID, region, connectClusterID, connectorID+"-gcs-sink", topics, gcsBucketName, tasksMax, formatOutputType, valueConverter, valueConverterSchemasEnable, keyConverter, gcsCredentialsDefault, options...); err != nil { t.Fatalf("failed to create Cloud Storage sink connector: %v", err) } got := buf.String() From 719bb760bfc4dd15d8044ae82e493df09789b5cf Mon Sep 17 00:00:00 2001 From: salmany Date: Fri, 12 Sep 2025 02:01:45 +0000 Subject: [PATCH 11/11] Fix CreateBigQuerySinkConnector parameters. --- managedkafka/connect/connectors/connectors_test.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/managedkafka/connect/connectors/connectors_test.go b/managedkafka/connect/connectors/connectors_test.go index d4d8cb20c1..4fceff7cc3 100644 --- a/managedkafka/connect/connectors/connectors_test.go +++ b/managedkafka/connect/connectors/connectors_test.go @@ -117,9 +117,8 @@ func TestConnectors(t *testing.T) { keyConverter := "org.apache.kafka.connect.storage.StringConverter" valueConverter := "org.apache.kafka.connect.json.JsonConverter" valueConverterSchemasEnable := "false" - gcsCredentialsDefault := "true" defaultDataset := "test-dataset" - if err := createBigQuerySinkConnector(buf, tc.ProjectID, region, connectClusterID, connectorID+"-bq-sink", topics, tasksMax, keyConverter, gcsCredentialsDefault, valueConverter, valueConverterSchemasEnable, defaultDataset, options...); err != nil { + if err := createBigQuerySinkConnector(buf, tc.ProjectID, region, connectClusterID, connectorID+"-bq-sink", topics, tasksMax, keyConverter, valueConverter, valueConverterSchemasEnable, defaultDataset, options...); err != nil { t.Fatalf("failed to create BigQuery sink connector: %v", err) } got := buf.String()