자체 관리형 W&B Weave 인스턴스 설정

helm repo add altinity https://helm.altinity.com
helm repo update

operator:
  image:
    repository: altinity/clickhouse-operator

  # Security context - 클러스터 요구 사항에 맞게 조정하세요
  containerSecurityContext:
    runAsGroup: 0
    runAsNonRoot: true
    runAsUser: 10001 # OpenShift/Kubernetes 보안 정책에 맞게 업데이트하세요
    allowPrivilegeEscalation: false
    capabilities:
      drop:
        - ALL
    privileged: false
    readOnlyRootFilesystem: false

metrics:
  enabled: false

# Name override - 필요한 경우 사용자 지정하세요
nameOverride: "wandb"

helm upgrade --install ch-operator altinity/altinity-clickhouse-operator \
  --namespace clickhouse \
  --create-namespace \
  -f ch-operator.yaml

# operator 파드가 실행 중인지 확인
kubectl get pods -n clickhouse

# 예상 출력:
# NAME                                 READY   STATUS    RESTARTS   AGE
# ch-operator-wandb-xxxxx              1/1     Running   0          30s

# operator 이미지 버전 확인
kubectl get pods -n clickhouse -o jsonpath="{.items[*].spec.containers[*].image}" | \
  tr ' ' '\n' | grep -v 'metrics-exporter' | sort -u

# 예상 출력:
# altinity/clickhouse-operator:0.25.4

# AWS 예시
aws s3 mb s3://[BUCKET-NAME] --region [REGION]

# ch-server.yaml에서 다음을 설정하세요:
<use_environment_credentials>true</use_environment_credentials>

{
  "Version": "2012-10-17",
  "Statement": [
    {
      "Effect": "Allow",
      "Action": [
        "s3:GetObject",
        "s3:PutObject",
        "s3:DeleteObject",
        "s3:ListBucket"
      ],
      "Resource": [
        "arn:aws:s3:::[BUCKET-NAME]",
        "arn:aws:s3:::[BUCKET-NAME]/*"
      ]
    }
  ]
}

kubectl create secret generic aws-creds \
  --namespace clickhouse \
  --from-literal aws_access_key=[ACCESS-KEY] \
  --from-literal aws_secret_key=[SECRET-KEY]

apiVersion: "clickhouse-keeper.altinity.com/v1"
kind: "ClickHouseKeeperInstallation"
metadata:
  name: wandb
  namespace: clickhouse
  annotations: {}
spec:
  defaults:
    templates:
      podTemplate: default
      dataVolumeClaimTemplate: default

  templates:
    podTemplates:
      - name: keeper
        metadata:
          labels:
            app: clickhouse-keeper
        spec:
          # Pod 보안 컨텍스트 - 환경에 맞게 조정하세요
          securityContext:
            fsGroup: 10001 # 클러스터의 보안 요구 사항에 따라 업데이트하세요
            fsGroupChangePolicy: Always
            runAsGroup: 0
            runAsNonRoot: true
            runAsUser: 10001 # OpenShift의 경우, 프로젝트에 할당된 UID 범위를 사용하세요
            seccompProfile:
              type: RuntimeDefault

          # 노드 전반에 keeper를 분산하기 위한 안티 어피니티 (HA에 권장)
          # 클러스터 크기 및 가용성 요구 사항에 따라 사용자 지정하거나 제거하세요
          affinity:
            podAntiAffinity:
              requiredDuringSchedulingIgnoredDuringExecution:
                - labelSelector:
                    matchExpressions:
                      - key: "app"
                        operator: In
                        values:
                          - clickhouse-keeper
                  topologyKey: "kubernetes.io/hostname"

          containers:
            - name: clickhouse-keeper
              imagePullPolicy: IfNotPresent
              image: "clickhouse/clickhouse-keeper:25.10"
              # 리소스 요청 - 예시 값이며, 워크로드에 따라 조정하세요
              resources:
                requests:
                  memory: "256Mi"
                  cpu: "0.5"
                limits:
                  memory: "2Gi"
                  cpu: "1"

              securityContext:
                allowPrivilegeEscalation: false
                capabilities:
                  drop:
                    - ALL
                privileged: false
                readOnlyRootFilesystem: false

    volumeClaimTemplates:
      - name: data
        metadata:
          labels:
            app: clickhouse-keeper
        spec:
          storageClassName: gp3 # 사용 중인 StorageClass로 변경하세요
          accessModes:
            - ReadWriteOnce
          resources:
            requests:
              storage: 10Gi

  configuration:
    clusters:
      - name: keeper # Keeper 클러스터 이름 - 서비스 DNS 명명에 사용됩니다
        layout:
          replicasCount: 3
        templates:
          podTemplate: keeper
          dataVolumeClaimTemplate: data

    settings:
      logger/level: "information"
      logger/console: "true"
      listen_host: "0.0.0.0"
      keeper_server/four_letter_word_white_list: "*"
      keeper_server/coordination_settings/raft_logs_level: "information"
      keeper_server/enable_ipv6: "false"
      keeper_server/coordination_settings/async_replication: "true"

kubectl apply -f ch-keeper.yaml

# Keeper 파드 확인
kubectl get pods -n clickhouse -l app=clickhouse-keeper

# 예상 출력:
# NAME                     READY   STATUS    RESTARTS   AGE
# chk-wandb-keeper-0-0-0   1/1     Running   0          2m
# chk-wandb-keeper-0-1-0   1/1     Running   0          2m
# chk-wandb-keeper-0-2-0   1/1     Running   0          2m

# Keeper 서비스 확인
kubectl get svc -n clickhouse | grep keeper

# 포트 2181에서 keeper 서비스가 표시되어야 합니다

apiVersion: "clickhouse.altinity.com/v1"
kind: "ClickHouseInstallation"
metadata:
  name: wandb
  namespace: clickhouse
  annotations: {}
spec:
  defaults:
    templates:
      podTemplate: default
      dataVolumeClaimTemplate: default

  templates:
    podTemplates:
      - name: clickhouse
        metadata:
          labels:
            app: clickhouse-server
        spec:
          # Pod security context - customize for your environment
          securityContext:
            fsGroup: 10001 # Adjust based on your security policies
            fsGroupChangePolicy: Always
            runAsGroup: 0
            runAsNonRoot: true
            runAsUser: 10001 # For OpenShift, use assigned UID range
            seccompProfile:
              type: RuntimeDefault

          # Anti-affinity rule - ensures servers run on different nodes (optional but recommended)
          # Adjust or remove based on your cluster size and requirements
          affinity:
            podAntiAffinity:
              requiredDuringSchedulingIgnoredDuringExecution:
                - labelSelector:
                    matchExpressions:
                      - key: "app"
                        operator: In
                        values:
                          - clickhouse-server
                  topologyKey: "kubernetes.io/hostname"

          containers:
            - name: clickhouse
              image: clickhouse/clickhouse-server:25.10
              # Example resource allocation - adjust based on workload
              resources:
                requests:
                  memory: 1Gi
                  cpu: 1
                limits:
                  memory: 16Gi
                  cpu: 4

              # AWS credentials (remove this section if using IRSA)
              env:
                - name: AWS_ACCESS_KEY_ID
                  valueFrom:
                    secretKeyRef:
                      name: aws-creds
                      key: aws_access_key
                - name: AWS_SECRET_ACCESS_KEY
                  valueFrom:
                    secretKeyRef:
                      name: aws-creds
                      key: aws_secret_key

              securityContext:
                allowPrivilegeEscalation: false
                capabilities:
                  drop:
                    - ALL
                privileged: false
                readOnlyRootFilesystem: false

    volumeClaimTemplates:
      - name: data
        metadata:
          labels:
            app: clickhouse-server
        spec:
          accessModes:
            - ReadWriteOnce
          resources:
            requests:
              storage: 50Gi
          storageClassName: gp3 # Change to your StorageClass

  configuration:
    # Keeper (ZooKeeper) configuration
    # IMPORTANT: These hostnames MUST match your Keeper deployment from Step 3
    zookeeper:
      nodes:
        - host: chk-wandb-keeper-0-0.clickhouse.svc.cluster.local
          port: 2181
        - host: chk-wandb-keeper-0-1.clickhouse.svc.cluster.local
          port: 2181
        - host: chk-wandb-keeper-0-2.clickhouse.svc.cluster.local
          port: 2181
      # Optional: Uncomment to adjust timeouts if needed
      # session_timeout_ms: 30000
      # operation_timeout_ms: 10000

    # Users configuration: https://clickhouse.com/docs/operations/configuration-files#user-settings
    # For production, use a SHA-256 hashed password instead of plain text:
    # printf "your-password" | sha256sum
    # Then use: weave/password_sha256_hex: <hash> instead of weave/password
    users:
      weave/password: [WEAVE-PASSWORD]  # 배포 전에 강력한 비밀번호로 교체하세요
      weave/access_management: 1
      weave/profile: default
      weave/networks/ip:
        - "0.0.0.0/0"
        - "::"

    # Server settings
    settings:
      disable_internal_dns_cache: 1

    # Cluster configuration
    clusters:
      - name: weavecluster # Cluster name - can be customized but must match wandb-cr.yaml
        layout:
          shardsCount: 1
          replicasCount: 3 # Number of replicas - adjust based on HA requirements
        templates:
          podTemplate: clickhouse
          dataVolumeClaimTemplate: data

    # Configuration files
    files:
      config.d/network_configuration.xml: |
        <clickhouse>
            <listen_host>0.0.0.0</listen_host>
            <listen_host>::</listen_host>
        </clickhouse>

      config.d/logger.xml: |
        <clickhouse>
            <logger>
                <level>information</level>
            </logger>
        </clickhouse>

      config.d/storage_configuration.xml: |
        <clickhouse>
            <storage_configuration>
                <disks>
                    <s3_disk>
                        <type>s3</type>
                        <!-- S3 버킷 엔드포인트 및 리전으로 업데이트하세요 -->
                        <endpoint>https://[BUCKET-NAME].s3.[REGION].amazonaws.com/s3_disk/{replica}</endpoint>
                        <metadata_path>/var/lib/clickhouse/disks/s3_disk/</metadata_path>
                        <use_environment_credentials>true</use_environment_credentials>
                        <region>[REGION]</region>
                    </s3_disk>
                    <s3_disk_cache>
                        <type>cache</type>
                        <disk>s3_disk</disk>
                        <path>/var/lib/clickhouse/s3_disk_cache/cache/</path>
                        <!-- Cache size MUST be smaller than persistent volume -->
                        <max_size>40Gi</max_size>
                        <cache_on_write_operations>true</cache_on_write_operations>
                    </s3_disk_cache>
                </disks>
                <policies>
                    <s3_main>
                        <volumes>
                            <main>
                                <disk>s3_disk_cache</disk>
                            </main>
                        </volumes>
                    </s3_main>
                </policies>
            </storage_configuration>
            <merge_tree>
                <storage_policy>s3_main</storage_policy>
            </merge_tree>
        </clickhouse>

<endpoint>https://my-wandb-clickhouse.s3.eu-central-1.amazonaws.com/s3_disk/{replica}</endpoint>
<region>eu-central-1</region>

<endpoint>https://minio.example.com:9000/my-bucket/s3_disk/{replica}</endpoint>
<region>us-east-1</region>

env:
  - name: AWS_ACCESS_KEY_ID
    valueFrom:
      secretKeyRef:
        name: aws-creds
        key: aws_access_key
  - name: AWS_SECRET_ACCESS_KEY
    valueFrom:
      secretKeyRef:
        name: aws-creds
        key: aws_secret_key

chk-wandb-keeper-0-0.clickhouse.svc.cluster.local
chk-wandb-keeper-0-1.clickhouse.svc.cluster.local
chk-wandb-keeper-0-2.clickhouse.svc.cluster.local

chk-myweave-keeper-0-0.clickhouse.svc.cluster.local
chk-myweave-keeper-0-1.clickhouse.svc.cluster.local
chk-myweave-keeper-0-2.clickhouse.svc.cluster.local

chk-wandb-coordination-0-0.clickhouse.svc.cluster.local
chk-wandb-coordination-0-1.clickhouse.svc.cluster.local
chk-wandb-coordination-0-2.clickhouse.svc.cluster.local

# 실제 이름을 확인하기 위해 Keeper 서비스 목록 조회
kubectl get svc -n clickhouse | grep keeper

# 명명 패턴을 확인하기 위해 Keeper 파드 목록 조회
kubectl get pods -n clickhouse -l app=clickhouse-keeper

kubectl apply -f ch-server.yaml

# ClickHouse 파드 확인
kubectl get pods -n clickhouse -l app=clickhouse-server

# 예상 출력:
# NAME                           READY   STATUS    RESTARTS   AGE
# chi-wandb-weavecluster-0-0-0   1/1     Running   0          3m
# chi-wandb-weavecluster-0-1-0   1/1     Running   0          3m
# chi-wandb-weavecluster-0-2-0   1/1     Running   0          3m

# ClickHouse 연결 테스트
kubectl exec -n clickhouse chi-wandb-weavecluster-0-0-0 -- \
  clickhouse-client --user weave --password [WEAVE-PASSWORD] --query "SELECT version()"

# 클러스터 상태 확인
kubectl exec -n clickhouse chi-wandb-weavecluster-0-0-0 -- \
  clickhouse-client --user weave --password [WEAVE-PASSWORD] --query \
  "SELECT cluster, host_name, port FROM system.clusters WHERE cluster='weavecluster'"

apiVersion: apps.wandb.com/v1
kind: WeightsAndBiases
metadata:
  name: wandb
  namespace: wandb
spec:
  values:
    global:
      # ... 기존 설정 ...

      # ClickHouse 설정 추가
      clickhouse:
        install: false # 별도로 배포함
        host: clickhouse-wandb.clickhouse.svc.cluster.local
        port: 8123
        user: weave
        password: [WEAVE-PASSWORD]
        database: weave
        replicated: true # 다중 레플리카 구성에 필수

      # Weave Trace 활성화
      weave-trace:
        enabled: true

    # Weave Trace 설정
    weave-trace:
      install: true
      extraEnv:
        WF_CLICKHOUSE_REPLICATED: "true"
        WF_CLICKHOUSE_REPLICATED_CLUSTER: "weavecluster"
      image:
        repository: wandb/weave-trace
        tag: 0.74.1
      replicaCount: 1
      size: "default"
      sizing:
        default:
          autoscaling:
            horizontal:
              enabled: false
          # 리소스 할당 예시 - 워크로드에 맞게 조정
          resources:
            limits:
              cpu: 4
              memory: "8Gi"
            requests:
              cpu: 1
              memory: "4Gi"
      # 파드 보안 컨텍스트 - 환경에 맞게 사용자 지정
      podSecurityContext:
        fsGroup: 10001 # 보안 요구 사항에 맞게 조정
        fsGroupChangePolicy: Always
        runAsGroup: 0
        runAsNonRoot: true
        runAsUser: 10001 # OpenShift의 경우 할당된 UID 범위 사용
        seccompProfile:
          type: RuntimeDefault
      # 컨테이너 보안 컨텍스트
      securityContext:
        allowPrivilegeEscalation: false
        capabilities:
          drop:
            - ALL
        privileged: false
        readOnlyRootFilesystem: false

kubectl apply -f wandb-cr.yaml

# weave-trace 파드 상태 확인
kubectl get pods -n wandb | grep weave-trace

# 예상 출력:
# wandb-weave-trace-bc-xxxxx   1/1     Running   0          2m

# ClickHouse 연결 관련 weave-trace 로그 확인
kubectl logs -n wandb [WEAVE-TRACE-POD-NAME] --tail=50

# ClickHouse 연결 성공 메시지 확인

# 시작 중 weave-trace 로그 확인
kubectl logs -n wandb [WEAVE-TRACE-POD-NAME] -f

# 데이터베이스 초기화 성공을 나타내는 마이그레이션 메시지 확인

# ClickHouse에 연결하고 데이터베이스 확인
kubectl exec -n clickhouse chi-wandb-weavecluster-0-0-0 -- \
  clickhouse-client --user weave --password [WEAVE-PASSWORD] --query \
  "SHOW DATABASES"

# 'weave' 데이터베이스가 목록에 표시되어야 함

# weave 데이터베이스의 테이블 확인
kubectl exec -n clickhouse chi-wandb-weavecluster-0-0-0 -- \
  clickhouse-client --user weave --password [WEAVE-PASSWORD] --query \
  "SHOW TABLES FROM weave"

import os
import weave

# Point Weave at your self-managed W&B instance
os.environ["WANDB_BASE_URL"] = "https://[WANDB-HOST]"  # W&B URL로 교체하세요

weave.init('test-project')

# Create a simple traced function
@weave.op()
def hello_weave(name: str) -> str:
    return f"Hello, {name}!"

# Call the function
result = hello_weave("World")
print(result)

kubectl get pvc -n clickhouse
kubectl describe pvc -n clickhouse

# 안티 어피니티 규칙이 스케줄링을 방해하는지 확인
kubectl describe pod -n clickhouse [POD-NAME] | grep -A 10 "Events:"

# 사용 가능한 노드 및 리소스 확인
kubectl get nodes
kubectl describe nodes | grep -A 5 "Allocated resources"

kubectl logs -n clickhouse [KEEPER-POD-NAME]

# 시크릿이 존재하는지 확인 (액세스 키 사용 시)
kubectl get secret aws-creds -n clickhouse

# S3 오류에 대한 ClickHouse 로그 확인
kubectl logs -n clickhouse [CLICKHOUSE-POD-NAME] | grep -i s3

# 저장소 설정에서 S3 엔드포인트 확인
kubectl get chi wandb -n clickhouse -o yaml | grep -A 10 storage_configuration

# Keeper 서비스 및 실제 이름 확인
kubectl get svc -n clickhouse | grep keeper

# Keeper 파드에서 명명 패턴 확인
kubectl get pods -n clickhouse -l app=clickhouse-keeper

# ch-server.yaml의 zookeeper.nodes 설정과 비교
# 호스트 이름은 실제 서비스 이름과 반드시 일치해야 함

# ClickHouse 로그에서 연결 오류 확인
kubectl logs -n clickhouse chi-wandb-weavecluster-0-0-0 | grep -i keeper

# weave-trace 파드 이름 조회
kubectl get pods -n wandb | grep weave-trace

# weave-trace 로그 확인
kubectl logs -n wandb [WEAVE-TRACE-POD-NAME]

# 일반적인 오류: "connection refused" 또는 "authentication failed"
# wandb-cr.yaml의 ClickHouse 자격 증명이 ch-server.yaml과 일치하는지 확인

# ch-server.yaml에서:
clusters:
  - name: weavecluster # <-- 이 이름

# wandb-cr.yaml에서 일치해야 함:
weave-trace:
  extraEnv:
    WF_CLICKHOUSE_REPLICATED_CLUSTER: "weavecluster" # <-- 이 값

image: clickhouse/clickhouse-keeper:25.10   # Keeper 버전
image: clickhouse/clickhouse-server:25.10   # 서버 버전

# 디스크 사용량 확인
kubectl exec -n clickhouse chi-wandb-weavecluster-0-0-0 -- \
  clickhouse-client --user weave --password [WEAVE-PASSWORD] --query \
  "SELECT name, path, formatReadableSize(free_space) as free, formatReadableSize(total_space) as total FROM system.disks"

# 복제 상태 확인
kubectl exec -n clickhouse chi-wandb-weavecluster-0-0-0 -- \
  clickhouse-client --user weave --password [WEAVE-PASSWORD] --query \
  "SELECT database, table, is_leader, total_replicas, active_replicas FROM system.replicas WHERE database='weave'"

# ClickHouse 서버 상태 확인
kubectl get pods -n clickhouse -l app=clickhouse-server

helm upgrade ch-operator altinity/altinity-clickhouse-operator \
  --namespace clickhouse \
  -f ch-operator.yaml

# ch-server.yaml 편집, image tag 변경
kubectl apply -f ch-server.yaml

# 파드 모니터링
kubectl get pods -n clickhouse

kubectl apply -f wandb-cr.yaml

# weave-trace 파드 재시작 모니터링
kubectl get pods -n wandb | grep weave-trace