Kubernetes NiFi Cluster setup in AKS
Please comment if you see any issues anywhere in the configuration or if you would like to provide any suggestions. Increase disk storage configuration according to your usage.
apiVersion: v1kind: Servicemetadata: name: nifi-hs labels: app: nifispec: ports: - port: 1025 name: nodeport - port: 8080 name: client clusterIP: None selector: app: nifi---apiVersion: v1kind: Servicemetadata: name: nifi-cs labels: app: nifi annotations: service.beta.kubernetes.io/azure-dns-label-name: nifispec: ports: - port: 80 targetPort: 8080 name: client selector: app: nifi type: LoadBalancer---apiVersion: policy/v1beta1kind: PodDisruptionBudgetmetadata: name: nifi-pdbspec: selector: matchLabels: app: nifi maxUnavailable: 1---apiVersion: storage.k8s.io/v1kind: StorageClassmetadata: name: nifi-sc selfLink: /apis/storage.k8s.io/v1/storageclasses/nifi-sc labels: addonmanager.kubernetes.io/mode: EnsureExists kubernetes.io/cluster-service: 'true'provisioner: kubernetes.io/azure-diskparameters: cachingmode: ReadOnly kind: Managed storageaccounttype: StandardSSD_LRSreclaimPolicy: DeleteallowVolumeExpansion: truevolumeBindingMode: WaitForFirstConsumer---apiVersion: apps/v1kind: StatefulSetmetadata: name: nifispec: selector: matchLabels: app: nifi serviceName: nifi-hs replicas: 3 updateStrategy: type: RollingUpdate podManagementPolicy: OrderedReady template: metadata: labels: app: nifi spec: affinity: podAntiAffinity: requiredDuringSchedulingIgnoredDuringExecution: - labelSelector: matchExpressions: - k: "app" operator: In values: - nifi topologyKey: "kubernetes.io/hostname" containers: - name: nifi image: "apache/nifi:1.13.0" env: - name: NIFI_CLUSTER_IS_NODE value: "true" - name: HOSTNAME valueFrom: fieldRef: fieldPath: metadata.name - name: NIFI_CLUSTER_ADDRESS value: $(HOSTNAME).nifi-hs - name: NIFI_CLUSTER_NODE_PROTOCOL_PORT value: "1025" - name: NIFI_WEB_HTTP_HOST value: $(HOSTNAME).nifi-hs.ns1.svc.cluster.local #- name: NIFI_WEB_HTTP_PORT # value: "80" - name: NIFI_CLUSTER_NODE_PROTOCOL_MAX_THREADS value: "100" - name: NIFI_ZK_CONNECT_STRING value: "zk-cs:2181" - name: NIFI_ELECTION_MAX_CANDIDATES value: "3" ports: - containerPort: 8080 name: client - containerPort: 1025 name: nodeport volumeMounts: - name: nifi-database mountPath: "/opt/nifi/nifi-current/database_repository" - name: nifi-flowfile mountPath: "/opt/nifi/nifi-current/flowfile_repository" - name: nifi-content mountPath: "/opt/nifi/nifi-current/content_repository" - name: nifi-provenance mountPath: "/opt/nifi/nifi-current/provenance_repository" - name: nifi-state mountPath: "/opt/nifi/nifi-current/state" securityContext: runAsUser: 1000 runAsGroup: 1000 fsGroup: 1000 volumeClaimTemplates: - metadata: name: nifi-database spec: storageClassName: "nifi-sc" accessModes: [ "ReadWriteOnce" ] resources: requests: storage: 5Gi - metadata: name: nifi-flowfile spec: storageClassName: "nifi-sc" accessModes: [ "ReadWriteOnce" ] resources: requests: storage: 5Gi - metadata: name: nifi-content spec: storageClassName: "nifi-sc" accessModes: [ "ReadWriteOnce" ] resources: requests: storage: 5Gi - metadata: name: nifi-provenance spec: storageClassName: "nifi-sc" accessModes: [ "ReadWriteOnce" ] resources: requests: storage: 5Gi - metadata: name: nifi-state spec: storageClassName: "nifi-sc" accessModes: [ "ReadWriteOnce" ] resources: requests: storage: 5Gi
The labelselector in affinity block is missing few words. Below is the updated working yaml block for statefulset.
apiVersion: apps/v1kind: StatefulSetmetadata: name: nifispec: selector: matchLabels: app: nifi serviceName: nifi-hs replicas: 3 updateStrategy: type: RollingUpdate podManagementPolicy: OrderedReady template: metadata: labels: app: nifi spec: affinity: podAntiAffinity: requiredDuringSchedulingIgnoredDuringExecution: - labelSelector: matchExpressions: - key: "app" operator: In values: - nifi topologyKey: "kubernetes.io/hostname" containers: - name: nifi image: "apache/nifi:1.13.0" env: - name: NIFI_CLUSTER_IS_NODE value: "true" - name: HOSTNAME valueFrom: fieldRef: fieldPath: metadata.name - name: NIFI_CLUSTER_ADDRESS value: $(HOSTNAME).nifi-hs - name: NIFI_CLUSTER_NODE_PROTOCOL_PORT value: "1025" - name: NIFI_WEB_HTTP_HOST value: $(HOSTNAME).nifi-hs.ns1.svc.cluster.local #- name: NIFI_WEB_HTTP_PORT # value: "80" - name: NIFI_CLUSTER_NODE_PROTOCOL_MAX_THREADS value: "100" - name: NIFI_ZK_CONNECT_STRING value: "zk-cs:2181" - name: NIFI_ELECTION_MAX_CANDIDATES value: "3" ports: - containerPort: 8080 name: client - containerPort: 1025 name: nodeport volumeMounts: - name: nifi-database mountPath: "/opt/nifi/nifi-current/database_repository" - name: nifi-flowfile mountPath: "/opt/nifi/nifi-current/flowfile_repository" - name: nifi-content mountPath: "/opt/nifi/nifi-current/content_repository" - name: nifi-provenance mountPath: "/opt/nifi/nifi-current/provenance_repository" - name: nifi-state mountPath: "/opt/nifi/nifi-current/state" securityContext: runAsUser: 1000 runAsGroup: 1000 fsGroup: 1000 volumeClaimTemplates: - metadata: name: nifi-database spec: storageClassName: "nifi-sc" accessModes: [ "ReadWriteOnce" ] resources: requests: storage: 5Gi - metadata: name: nifi-flowfile spec: storageClassName: "nifi-sc" accessModes: [ "ReadWriteOnce" ] resources: requests: storage: 5Gi - metadata: name: nifi-content spec: storageClassName: "nifi-sc" accessModes: [ "ReadWriteOnce" ] resources: requests: storage: 5Gi - metadata: name: nifi-provenance spec: storageClassName: "nifi-sc" accessModes: [ "ReadWriteOnce" ] resources: requests: storage: 5Gi - metadata: name: nifi-state spec: storageClassName: "nifi-sc" accessModes: [ "ReadWriteOnce" ] resources: requests: storage: 5Gi