add: anowflake email kafka, refa: redis connectg

This commit is contained in:
wwweww
2026-02-25 01:16:13 +08:00
parent fdbcde13b2
commit 300058ad01
67 changed files with 3596 additions and 139 deletions
@@ -1,38 +1,38 @@
apiVersion: v1
kind: Namespace
metadata:
name: juwan
---
apiVersion: v1
kind: ServiceAccount
metadata:
namespace: juwan
name: find-endpoints
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: discov-endpoints
rules:
- apiGroups: [""]
resources: ["endpoints"]
verbs: ["get", "list", "watch"]
- apiGroups: ["discovery.k8s.io"]
resources: ["endpointslices"]
verbs: ["get", "list", "watch"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: find-endpoints-discov-endpoints
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: discov-endpoints
subjects:
- kind: ServiceAccount
name: find-endpoints
namespace: juwan
apiVersion: v1
kind: Namespace
metadata:
name: juwan
---
apiVersion: v1
kind: ServiceAccount
metadata:
namespace: juwan
name: find-endpoints
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: discov-endpoints
rules:
- apiGroups: [""]
resources: ["endpoints"]
verbs: ["get", "list", "watch"]
- apiGroups: ["discovery.k8s.io"]
resources: ["endpointslices"]
verbs: ["get", "list", "watch"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: find-endpoints-discov-endpoints
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: discov-endpoints
subjects:
- kind: ServiceAccount
name: find-endpoints
namespace: juwan
-11
View File
@@ -1,11 +0,0 @@
apiVersion: v1
kind: ConfigMap
metadata:
name: db-dx-init-script
namespace: juwan
labels:
app: db-dx-init-script
data:
init-extensions-sql: |
create extension if not exists "uuid-ossp";
create extension if not exists "pg_trgm";
+33
View File
@@ -0,0 +1,33 @@
apiVersion: v1
kind: Service
metadata:
name: snowflake-sve
namespace: juwan
spec:
ClusterIP: None
selector:
app: snowflake
ports:
- port: 9000
targetPort: 9000
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: snowflake
namespace: juwan
spec:
serviceName: snowflake-svc
replicas: 3
selector:
matchLabels:
app: snowflake
template:
metadata:
labels:
app: snowflake
spec:
containers:
- name: snowflake
image:
+75
View File
@@ -0,0 +1,75 @@
# apiVersion: kafka.strimzi.io/v1
# kind: KafkaNodePool
# metadata:
# name: kafka-pool
# namespace: kafka
# labels:
# strimzi.io/cluster: my-cluster
# spec:
# replicas: 3
# roles:
# - controller
# - broker
# storage:
# type: jbod
# volumes:
# - id: 0
# type: persistent-claim
# size: 100Gi
# deleteClaim: false
# resources:
# requests:
# memory: 2Gi
# cpu: "1"
# limits:
# memory: 4Gi
# cpu: "2"
# ---
apiVersion: kafka.strimzi.io/v1
kind: KafkaNodePool
metadata:
name: controller-pool
namespace: kafka
labels:
strimzi.io/cluster: my-cluster
spec:
replicas: 3
roles:
- controller
storage:
type: persistent-claim
size: 10Gi
deleteClaim: false
resources:
requests:
memory: 1Gi
cpu: "0.5"
limits:
memory: 2Gi
cpu: "1"
---
apiVersion: kafka.strimzi.io/v1
kind: KafkaNodePool
metadata:
name: broker-pool
namespace: kafka
labels:
strimzi.io/cluster: my-cluster
spec:
replicas: 3
roles:
- broker
storage:
type: jbod
volumes:
- id: 0
type: persistent-claim
size: 100Gi
deleteClaim: false
resources:
requests:
memory: 2Gi
cpu: "1"
limits:
memory: 4Gi
cpu: "2"
+44
View File
@@ -0,0 +1,44 @@
apiVersion: kafka.strimzi.io/v1
kind: Kafka
metadata:
name: my-cluster
namespace: kafka
annotations:
strimzi.io/kraft: enabled
strimzi.io/node-pools: enabled
spec:
kafka:
version: 4.0.1
metadataVersion: 4.0-IV0
listeners:
- name: plain
port: 9092
type: internal
tls: false
- name: tls
port: 9093
type: internal
tls: true
config:
offsets.topic.replication.factor: 3
transaction.state.log.replication.factor: 3
transaction.state.log.min.isr: 2
default.replication.factor: 3
min.insync.replicas: 2
entityOperator:
topicOperator:
resources:
requests:
memory: 512Mi
cpu: "0.2"
limits:
memory: 512Mi
cpu: "0.5"
userOperator:
resources:
requests:
memory: 512Mi
cpu: "0.2"
limits:
memory: 512Mi
cpu: "0.5"
+13
View File
@@ -0,0 +1,13 @@
apiVersion: kafka.strimzi.io/v1
kind: KafkaTopic
metadata:
name: email-task
namespace: kafka
labels:
strimzi.io/cluster: my-cluster
spec:
partitions: 3
replicas: 3
config:
retention.ms: 604800000
segment.bytes: 1073741824
+43 -2
View File
@@ -11,7 +11,11 @@ metadata:
rules:
- apiGroups: [""]
resources:
- nodes
- pods
- pods/log
- services
- endpoints
- namespaces
verbs: ["get", "list", "watch"]
---
@@ -50,6 +54,14 @@ data:
kubernetes_sd_configs:
- role: pod
relabel_configs:
- action: replace
source_labels: [__meta_kubernetes_pod_label_app_kubernetes_io_name]
target_label: app
regex: (.+)
- action: replace
source_labels: [__meta_kubernetes_pod_label_app]
target_label: app
regex: (.+)
- action: replace
source_labels: [__meta_kubernetes_pod_node_name]
target_label: node
@@ -63,9 +75,29 @@ data:
source_labels: [__meta_kubernetes_pod_container_name]
target_label: container
- action: replace
source_labels: [__meta_kubernetes_pod_uid]
source_labels: [__meta_kubernetes_pod_uid, __meta_kubernetes_pod_container_name]
separator: /
target_label: __path__
replacement: /var/log/pods/*$1/*/*.log
replacement: /var/log/pods/*$1/*.log
- job_name: kubernetes-pods-static
pipeline_stages:
- regex:
source: filename
expression: /var/log/pods/(?P<namespace>[^_]+)_(?P<pod>[^_]+)_[^/]+/(?P<container>[^/]+)/[0-9]+\.log
- regex:
source: pod
expression: ^(?P<app>.+?)(?:-[a-f0-9]{8,10}-[a-z0-9]{5}|-[0-9]+)?$
- labels:
namespace:
pod:
container:
app:
static_configs:
- targets:
- localhost
labels:
job: kubernetes-pods
__path__: /var/log/pods/*/*/*.log
---
apiVersion: apps/v1
kind: DaemonSet
@@ -87,6 +119,9 @@ spec:
containers:
- name: promtail
image: grafana/promtail:2.9.6
securityContext:
runAsUser: 0
runAsGroup: 0
args:
- "-config.file=/etc/promtail/promtail.yaml"
volumeMounts:
@@ -97,6 +132,9 @@ spec:
- name: varlog
mountPath: /var/log
readOnly: true
- name: dockercontainers
mountPath: /var/lib/docker/containers
readOnly: true
volumes:
- name: config
configMap:
@@ -106,3 +144,6 @@ spec:
- name: varlog
hostPath:
path: /var/log
- name: dockercontainers
hostPath:
path: /var/lib/docker/containers
+119
View File
@@ -0,0 +1,119 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: email-api
namespace: juwan
labels:
app: email-api
spec:
replicas: 3
revisionHistoryLimit: 5
selector:
matchLabels:
app: email-api
template:
metadata:
labels:
app: email-api
spec:
serviceAccountName: find-endpoints
containers:
- name: email-api
image: email
ports:
- containerPort: 8888
env:
- name: KAFKA_BROKER
value: "my-cluster-kafka-bootstrap.kafka.svc.cluster.local:9092"
- name: REDIS_M_HOST
value: "user-redis-master.juwan:6379"
- name: REDIS_S_HOST
value: "user-redis-replica.juwan:6379"
- name: REDIS_PASSWORD
valueFrom:
secretKeyRef:
name: user-redis
key: password
readinessProbe:
tcpSocket:
port: 8888
initialDelaySeconds: 5
periodSeconds: 10
livenessProbe:
tcpSocket:
port: 8888
initialDelaySeconds: 15
periodSeconds: 20
resources:
requests:
cpu: 500m
memory: 512Mi
limits:
cpu: 1000m
memory: 1024Mi
volumeMounts:
- name: timezone
mountPath: /etc/localtime
volumes:
- name: timezone
hostPath:
path: /usr/share/zoneinfo/Asia/Shanghai
---
apiVersion: v1
kind: Service
metadata:
name: email-api-svc
namespace: juwan
spec:
ports:
- port: 8888
targetPort: 8888
selector:
app: email-api
---
apiVersion: autoscaling/v2
kind: HorizontalPodAutoscaler
metadata:
name: email-api-hpa-c
namespace: juwan
labels:
app: email-api-hpa-c
spec:
scaleTargetRef:
apiVersion: apps/v1
kind: Deployment
name: email-api
minReplicas: 3
maxReplicas: 10
metrics:
- type: Resource
resource:
name: cpu
target:
type: Utilization
averageUtilization: 80
---
apiVersion: autoscaling/v2
kind: HorizontalPodAutoscaler
metadata:
name: email-api-hpa-m
namespace: juwan
labels:
app: email-api-hpa-m
spec:
scaleTargetRef:
apiVersion: apps/v1
kind: Deployment
name: email-api
minReplicas: 3
maxReplicas: 10
metrics:
- type: Resource
resource:
name: memory
target:
type: Utilization
averageUtilization: 80
+100
View File
@@ -0,0 +1,100 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: email-consumer
namespace: juwan
labels:
app: email-consumer
spec:
replicas: 3
revisionHistoryLimit: 5
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 0
maxUnavailable: 1
selector:
matchLabels:
app: email-consumer
template:
metadata:
labels:
app: email-consumer
spec:
serviceAccountName: find-endpoints
containers:
- name: email-consumer
image: 103.236.53.208:4418/library/email-consumer@sha256:6fe8a3a57310a5e79feecc4bf38ac2c5b8c58a7f200f104f7bf4707b9db5fc13
resources:
requests:
cpu: 100m
memory: 128Mi
limits:
cpu: 500m
memory: 512Mi
volumeMounts:
- name: timezone
mountPath: /etc/localtime
volumes:
- name: timezone
hostPath:
path: /usr/share/zoneinfo/Asia/Shanghai
---
apiVersion: v1
kind: Service
metadata:
name: email-consumer-svc
namespace: juwan
spec:
ports:
- port: 8080
targetPort: 8080
selector:
app: email-consumer
---
apiVersion: autoscaling/v2
kind: HorizontalPodAutoscaler
metadata:
name: email-consumer-hpa-c
namespace: juwan
labels:
app: email-consumer-hpa-c
spec:
scaleTargetRef:
apiVersion: apps/v1
kind: Deployment
name: email-consumer
minReplicas: 1
maxReplicas: 3
metrics:
- type: Resource
resource:
name: cpu
target:
type: Utilization
averageUtilization: 80
---
apiVersion: autoscaling/v2
kind: HorizontalPodAutoscaler
metadata:
name: email-consumer-hpa-m
namespace: juwan
labels:
app: email-consumer-hpa-m
spec:
scaleTargetRef:
apiVersion: apps/v1
kind: Deployment
name: email-consumer
minReplicas: 1
maxReplicas: 3
metrics:
- type: Resource
resource:
name: memory
target:
type: Utilization
averageUtilization: 80
+107
View File
@@ -0,0 +1,107 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: snowflake
namespace: juwan
labels:
app: snowflake
spec:
replicas: 3
revisionHistoryLimit: 5
selector:
matchLabels:
app: snowflake
template:
metadata:
labels:
app: snowflake
spec:
serviceAccountName: find-endpoints
containers:
- name: snowflake
image: 103.236.53.208:4418/library/snowflake@sha256:1679cf94b69f426eec5d2f960ffb153bb7dbcd3bcaf0286261a43756384a86b3
ports:
- containerPort: 8080
readinessProbe:
tcpSocket:
port: 8080
initialDelaySeconds: 5
periodSeconds: 10
livenessProbe:
tcpSocket:
port: 8080
initialDelaySeconds: 15
periodSeconds: 20
resources:
requests:
cpu: 500m
memory: 512Mi
limits:
cpu: 1000m
memory: 1024Mi
volumeMounts:
- name: timezone
mountPath: /etc/localtime
volumes:
- name: timezone
hostPath:
path: /usr/share/zoneinfo/Asia/Shanghai
---
apiVersion: v1
kind: Service
metadata:
name: snowflake-svc
namespace: juwan
spec:
ports:
- port: 8080
targetPort: 8080
selector:
app: snowflake
---
apiVersion: autoscaling/v2
kind: HorizontalPodAutoscaler
metadata:
name: snowflake-hpa-c
namespace: juwan
labels:
app: snowflake-hpa-c
spec:
scaleTargetRef:
apiVersion: apps/v1
kind: Deployment
name: snowflake
minReplicas: 3
maxReplicas: 10
metrics:
- type: Resource
resource:
name: cpu
target:
type: Utilization
averageUtilization: 80
---
apiVersion: autoscaling/v2
kind: HorizontalPodAutoscaler
metadata:
name: snowflake-hpa-m
namespace: juwan
labels:
app: snowflake-hpa-m
spec:
scaleTargetRef:
apiVersion: apps/v1
kind: Deployment
name: snowflake
minReplicas: 3
maxReplicas: 10
metrics:
- type: Resource
resource:
name: memory
target:
type: Utilization
averageUtilization: 80
+65 -11
View File
@@ -29,18 +29,35 @@ spec:
]
containers:
- name: user-rpc
image: user-rpc:v1
image: 103.236.53.208:4418/library/user-rpc@sha256:57746256905acb5757153aef536ebfd19338b7f935f01ba1f538fbfd0a12f6f5
ports:
- containerPort: 9001
- containerPort: 4001
env:
- name: DB_URI
- name: DB_PORT
valueFrom:
secretKeyRef:
name: user-db-app
key: uri
- name: REDIS_HOST
value: "user-redis.juwan:6379"
key: port
- name: DB_PASSWORD
valueFrom:
secretKeyRef:
name: user-db-app
key: password
- name: PD_USERNAME
valueFrom:
secretKeyRef:
name: user-db-app
key: username
- name: DB_NAME
valueFrom:
secretKeyRef:
name: user-db-app
key: dbname
- name: REDIS_M_HOST
value: "user-redis-master.juwan:6379"
- name: REDIS_S_HOST
value: "user-redis-replica.juwan:6379"
- name: REDIS_PASSWORD
valueFrom:
secretKeyRef:
@@ -143,9 +160,9 @@ spec:
type: Utilization
averageUtilization: 80
---
# Redis Cluster
# Redis 主从复制
apiVersion: redis.redis.opstreelabs.in/v1beta2
kind: RedisCluster
kind: RedisReplication
metadata:
name: user-redis
namespace: juwan
@@ -161,9 +178,10 @@ spec:
limits:
cpu: 500m
memory: 512Mi
redisSecret:
name: user-redis
key: password
redisSecret:
name: user-redis
key: password
redisExporter:
enabled: true
image: quay.io/opstree/redis-exporter:latest
@@ -172,7 +190,43 @@ spec:
runAsUser: 1000
fsGroup: 1000
storage:
size: 1Gi
volumeClaimTemplate:
spec:
accessModes: ["ReadWriteOnce"]
resources:
requests:
storage: 1Gi
---
# Sentinel 监控
apiVersion: redis.redis.opstreelabs.in/v1beta2
kind: RedisSentinel
metadata:
name: user-redis-sentinel
namespace: juwan
spec:
clusterSize: 3
kubernetesConfig:
image: quay.io/opstree/redis-sentinel:v7.0.12
imagePullPolicy: IfNotPresent
resources:
requests:
cpu: 100m
memory: 128Mi
limits:
cpu: 500m
memory: 512Mi
podSecurityContext:
runAsUser: 1000
fsGroup: 1000
redisSentinelConfig:
redisReplicationName: user-redis
masterGroupName: mymaster
redisPort: "6379"
quorum: "2"
downAfterMilliseconds: "5000"
failoverTimeout: "10000"
parallelSyncs: "1"
---
# PostgreSQL 集群