apiVersion: v1
kind: List
items:
  # This configuration file sets up a secure StatefulSet running CockroachDB with
  # tweaks to make it more performant than our default configuration files. All
  # changes from the default secure configuration have been marked with a comment
  # starting with "NOTE" or "TODO".
  #
  # To use it, customize all the parts of the file labeled "TODO" before running:
  #   kubectl create -f cockroachdb-statefulset-secure.yaml
  #
  # You will then have to approve certificate-signing requests and initialize the
  # cluster as described in the parent directory's README.md file.
  #
  # If you don't see any pods being created, it's possible that your cluster was
  # not able to meet the resource requests asked for, whether it was the amount
  # of CPU, memory, or disk or the disk type. To find information about why pods
  # haven't been created, you can run:
  #   kubectl get events
  #
  # For more information on improving CockroachDB performance in Kubernetes, see
  # our docs:
  # https://www.cockroachlabs.com/docs/stable/kubernetes-performance.html
  - apiVersion: v1
    kind: ServiceAccount
    metadata:
      name: cockroachdb
      labels:
        app: cockroachdb
  - apiVersion: rbac.authorization.k8s.io/v1
    kind: Role
    metadata:
      name: cockroachdb
      labels:
        app: cockroachdb
    rules:
    - apiGroups:
      - ""
      resources:
      - secrets
      verbs:
      - create
      - get
  - apiVersion: rbac.authorization.k8s.io/v1
    kind: ClusterRole
    metadata:
      name: cockroachdb
      labels:
        app: cockroachdb
    rules:
    - apiGroups:
      - certificates.k8s.io
      resources:
      - certificatesigningrequests
      verbs:
      - create
      - get
      - watch
  - apiVersion: rbac.authorization.k8s.io/v1
    kind: RoleBinding
    metadata:
      name: cockroachdb
      labels:
        app: cockroachdb
    roleRef:
      apiGroup: rbac.authorization.k8s.io
      kind: Role
      name: cockroachdb
    subjects:
    - kind: ServiceAccount
      name: cockroachdb
      namespace: default
  - apiVersion: rbac.authorization.k8s.io/v1
    kind: ClusterRoleBinding
    metadata:
      name: cockroachdb
      labels:
        app: cockroachdb
    roleRef:
      apiGroup: rbac.authorization.k8s.io
      kind: ClusterRole
      name: cockroachdb
    subjects:
    - kind: ServiceAccount
      name: cockroachdb
      namespace: default
  - apiVersion: v1
    kind: Service
    metadata:
      # This service is meant to be used by clients of the database. It exposes a ClusterIP that will
      # automatically load balance connections to the different database pods.
      name: cockroachdb-public
      labels:
        app: cockroachdb
    spec:
      ports:
      # The main port, served by gRPC, serves Postgres-flavor SQL, internode
      # traffic and the cli.
      - port: 26257
        targetPort: 26257
        name: grpc
      # The secondary port serves the UI as well as health and debug endpoints.
      - port: 8080
        targetPort: 8080
        name: http
      selector:
        app: cockroachdb
  - apiVersion: v1
    kind: Service
    metadata:
      # This service only exists to create DNS entries for each pod in the stateful
      # set such that they can resolve each other's IP addresses. It does not
      # create a load-balanced ClusterIP and should not be used directly by clients
      # in most circumstances.
      name: cockroachdb
      labels:
        app: cockroachdb
      annotations:
        # Use this annotation in addition to the actual publishNotReadyAddresses
        # field below because the annotation will stop being respected soon but the
        # field is broken in some versions of Kubernetes:
        # https://github.com/kubernetes/kubernetes/issues/58662
        service.alpha.kubernetes.io/tolerate-unready-endpoints: "true"
        # Enable automatic monitoring of all instances when Prometheus is running in the cluster.
        prometheus.io/scrape: "true"
        prometheus.io/path: "_status/vars"
        prometheus.io/port: "8080"
    spec:
      ports:
      - port: 26257
        targetPort: 26257
        name: grpc
      - port: 8080
        targetPort: 8080
        name: http
      # We want all pods in the StatefulSet to have their addresses published for
      # the sake of the other CockroachDB pods even before they're ready, since they
      # have to be able to talk to each other in order to become ready.
      publishNotReadyAddresses: true
      clusterIP: None
      selector:
        app: cockroachdb
  - apiVersion: policy/v1
    kind: PodDisruptionBudget
    metadata:
      name: cockroachdb-budget
      labels:
        app: cockroachdb
    spec:
      selector:
        matchLabels:
          app: cockroachdb
      maxUnavailable: 1
  - apiVersion: apps/v1
    kind: StatefulSet
    metadata:
      name: cockroachdb
    spec:
      serviceName: "cockroachdb"
      replicas: 3
      selector:
        matchLabels:
          app: cockroachdb
      template:
        metadata:
          labels:
            app: cockroachdb
        spec:
          serviceAccountName: cockroachdb
          # NOTE: Running with `hostNetwork: true` means that CockroachDB will use
          # the host machines' IP address and hostname, and that nothing else on
          # the machines will be able to use the same ports. This means that only 1
          # CockroachDB pod will ever be schedulable on the same machine, because
          # otherwise their ports would conflict.
          #
          # If your client pods generate a lot of network traffic to and from the
          # CockroachDB cluster, you may see a benefit to doing the same thing in
          # their configurations.
          hostNetwork: true
          dnsPolicy: ClusterFirstWithHostNet
          # Init containers are run only once in the lifetime of a pod, before
          # it's started up for the first time. It has to exit successfully
          # before the pod's main containers are allowed to start.
          initContainers:
          # The init-certs container sends a certificate signing request to the
          # kubernetes cluster.
          # You can see pending requests using: kubectl get csr
          # CSRs can be approved using:         kubectl certificate approve <csr name>
          #
          # All addresses used to contact a node must be specified in the --addresses arg.
          #
          # In addition to the node certificate and key, the init-certs entrypoint will symlink
          # the cluster CA to the certs directory.
          - name: init-certs
            image: cockroachdb/cockroach-k8s-request-cert:0.4
            imagePullPolicy: IfNotPresent
            command:
            - "/bin/ash"
            - "-ecx"
            - "/request-cert -namespace=${POD_NAMESPACE} -certs-dir=/cockroach-certs -type=node -addresses=localhost,127.0.0.1,$(hostname -f),$(hostname -f|cut -f 1-2 -d '.'),cockroachdb-public,cockroachdb-public.$(hostname -f|cut -f 3- -d '.'),cockroachdb-public.$(hostname -f|cut -f 3-4 -d '.'),cockroachdb-public.$(hostname -f|cut -f 3 -d '.') -symlink-ca-from=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
            env:
            - name: POD_NAMESPACE
              valueFrom:
                fieldRef:
                  fieldPath: metadata.namespace
            volumeMounts:
            - name: certs
              mountPath: /cockroach-certs
          # NOTE: If you are running clients that generate heavy load, you may find
          # it useful to copy this anti-affinity policy into the client pods'
          # configurations as well to avoid running them on the same machines as
          # CockroachDB and interfering with each other's performance.
          affinity:
            podAntiAffinity:
              preferredDuringSchedulingIgnoredDuringExecution:
              - weight: 100
                podAffinityTerm:
                  labelSelector:
                    matchExpressions:
                    - key: app
                      operator: In
                      values:
                      - cockroachdb
                  topologyKey: kubernetes.io/hostname
          containers:
          - name: cockroachdb
            # NOTE: Always use the most recent version of CockroachDB for the best
            # performance and reliability.
            image: cockroachdb/cockroach:latest
            imagePullPolicy: IfNotPresent
            # TODO: Change these to appropriate values for the hardware that you're running. You can see
            # the resources that can be allocated on each of your Kubernetes nodes by running:
            #   kubectl describe nodes
            # Note that requests and limits should have identical values.
            resources:
              requests:
                cpu: "2"
                memory: "8Gi"
              limits:
                cpu: "2"
                memory: "8Gi" 
            ports:
            - containerPort: 26257
              name: grpc
            - containerPort: 8080
              name: http
    # We recommend that you do not configure a liveness probe on a production environment, as this can impact the availability of production databases.
    #       livenessProbe:
    #         httpGet:
    #           path: "/health"
    #           port: http
    #           scheme: HTTPS
    #         initialDelaySeconds: 30
    #         periodSeconds: 5
            readinessProbe:
              httpGet:
                path: "/health?ready=1"
                port: http
                scheme: HTTPS
              initialDelaySeconds: 10
              periodSeconds: 5
              failureThreshold: 2
            volumeMounts:
            - name: datadir
              mountPath: /cockroach/cockroach-data
            - name: certs
              mountPath: /cockroach/cockroach-certs
            env:
            - name: COCKROACH_CHANNEL
              value: kubernetes-secure
            command:
              - "/bin/bash"
              - "-ecx"
              # The use of qualified `hostname -f` is crucial:
              # Other nodes aren't able to look up the unqualified hostname.
              - "exec /cockroach/cockroach start --logtostderr --certs-dir /cockroach/cockroach-certs --advertise-host $(hostname -f) --http-addr 0.0.0.0 --join cockroachdb-0.cockroachdb,cockroachdb-1.cockroachdb,cockroachdb-2.cockroachdb --cache 25% --max-sql-memory 25%"
          # No pre-stop hook is required, a SIGTERM plus some time is all that's
          # needed for graceful shutdown of a node.
          terminationGracePeriodSeconds: 60
          volumes:
          - name: datadir
            persistentVolumeClaim:
              claimName: datadir
          - name: certs
            emptyDir: {}
      podManagementPolicy: Parallel
      updateStrategy:
        type: RollingUpdate
      volumeClaimTemplates:
      - metadata:
          name: datadir
        spec:
          accessModes:
            - "ReadWriteOnce"
          # TODO: This specifically asks for a storage class with the name "ssd". A
          # storage class of this name doesn't exist by default. See our docs for
          # more information on how to create an optimized storage class for use here:
          # https://www.cockroachlabs.com/docs/stable/kubernetes-performance.html#disk-type
          storageClassName: ssd
          resources:
            requests:
              # TODO: This asks for a fairly large disk by default because on
              # certain popular clouds there is a direct correlation between disk
              # size and the IOPS provisioned to the disk. Change this as necessary
              # to suit your needs, but be aware that smaller disks will typically
              # mean worse performance.
              storage: 1024Gi
