diff options
16 files changed, 655 insertions, 61 deletions
diff --git a/roles/openshift_storage_glusterfs/README.md b/roles/openshift_storage_glusterfs/README.md index abe411f67..03c157313 100644 --- a/roles/openshift_storage_glusterfs/README.md +++ b/roles/openshift_storage_glusterfs/README.md @@ -84,6 +84,20 @@ GlusterFS cluster into a new or existing OpenShift cluster: | openshift_storage_glusterfs_storageclass | True | Automatically create a StorageClass for each GlusterFS cluster | openshift_storage_glusterfs_image | 'gluster/gluster-centos'| Container image to use for GlusterFS pods, enterprise default is 'rhgs3/rhgs-server-rhel7' | openshift_storage_glusterfs_version | 'latest' | Container image version to use for GlusterFS pods +| openshift_storage_glusterfs_block_deploy | True | Deploy glusterblock provisioner service +| openshift_storage_glusterfs_block_image | 'gluster/glusterblock-provisioner'| Container image to use for glusterblock-provisioner pod, enterprise default is 'rhgs3/rhgs-gluster-block-prov-rhel7' +| openshift_storage_glusterfs_block_version | 'latest' | Container image version to use for glusterblock-provisioner pod +| openshift_storage_glusterfs_block_max_host_vol | 15 | Max number of GlusterFS volumes to host glusterblock volumes +| openshift_storage_glusterfs_s3_deploy | True | Deploy gluster-s3 service +| openshift_storage_glusterfs_s3_image | 'gluster/gluster-object'| Container image to use for gluster-s3 pod, enterprise default is 'rhgs3/rhgs-gluster-s3-server-rhel7' +| openshift_storage_glusterfs_s3_version | 'latest' | Container image version to use for gluster=s3 pod +| openshift_storage_glusterfs_s3_account | Undefined | S3 account name for the S3 service, required for S3 service deployment +| openshift_storage_glusterfs_s3_user | Undefined | S3 user name for the S3 service, required for S3 service deployment +| openshift_storage_glusterfs_s3_password | Undefined | S3 user password for the S3 service, required for S3 service deployment +| openshift_storage_glusterfs_s3_pvc | Dynamic | Name of the GlusterFS-backed PVC which will be used for S3 object data storage, generated from the cluster name and S3 account by default +| openshift_storage_glusterfs_s3_pvc_size | "2Gi" | Size, in Gi, of the GlusterFS-backed PVC which will be used for S3 object data storage +| openshift_storage_glusterfs_s3_meta_pvc | Dynamic | Name of the GlusterFS-backed PVC which will be used for S3 object metadata storage, generated from the cluster name and S3 account by default +| openshift_storage_glusterfs_s3_meta_pvc_size | "1Gi" | Size, in Gi, of the GlusterFS-backed PVC which will be used for S3 object metadata storage | openshift_storage_glusterfs_wipe | False | Destroy any existing GlusterFS resources and wipe storage devices. **WARNING: THIS WILL DESTROY ANY DATA ON THOSE DEVICES.** | openshift_storage_glusterfs_heketi_is_native | True | heketi should be containerized | openshift_storage_glusterfs_heketi_cli | 'heketi-cli' | Command/Path to invoke the heketi-cli tool **NOTE:** Change this only for **non-native heketi** if heketi-cli is not in the global `$PATH` of the machine running openshift-ansible @@ -99,6 +113,7 @@ GlusterFS cluster into a new or existing OpenShift cluster: | openshift_storage_glusterfs_heketi_ssh_user | 'root' | SSH user for external GlusterFS nodes via native heketi | openshift_storage_glusterfs_heketi_ssh_sudo | False | Whether to sudo (if non-root user) for SSH to external GlusterFS nodes via native heketi | openshift_storage_glusterfs_heketi_ssh_keyfile | Undefined | Path to a private key file for use with SSH connections to external GlusterFS nodes via native heketi **NOTE:** This must be an absolute path +| openshift_storage_glusterfs_heketi_fstab | '/var/lib/heketi/fstab' | When heketi is native, sets the path to the fstab file on the GlusterFS nodes to update on LVM volume mounts, changes to '/etc/fstab/' when the heketi executor is 'ssh' **NOTE:** This should not need to be changed | openshift_storage_glusterfs_heketi_wipe | False | Destroy any existing heketi resources, defaults to the value of `openshift_storage_glusterfs_wipe` Each role variable also has a corresponding variable to optionally configure a diff --git a/roles/openshift_storage_glusterfs/defaults/main.yml b/roles/openshift_storage_glusterfs/defaults/main.yml index 148549887..c3db36d37 100644 --- a/roles/openshift_storage_glusterfs/defaults/main.yml +++ b/roles/openshift_storage_glusterfs/defaults/main.yml @@ -7,6 +7,20 @@ openshift_storage_glusterfs_use_default_selector: False openshift_storage_glusterfs_storageclass: True openshift_storage_glusterfs_image: "{{ 'rhgs3/rhgs-server-rhel7' | quote if deployment_type == 'openshift-enterprise' else 'gluster/gluster-centos' | quote }}" openshift_storage_glusterfs_version: 'latest' +openshift_storage_glusterfs_block_deploy: True +openshift_storage_glusterfs_block_image: "{{ 'rhgs3/rhgs-gluster-block-prov-rhel7' | quote if deployment_type == 'openshift-enterprise' else 'gluster/glusterblock-provisioner' | quote }}" +openshift_storage_glusterfs_block_version: 'latest' +openshift_storage_glusterfs_block_max_host_vol: 15 +openshift_storage_glusterfs_s3_deploy: True +openshift_storage_glusterfs_s3_image: "{{ 'rhgs3/rhgs-gluster-s3-server-rhel7' | quote if deployment_type == 'openshift-enterprise' else 'gluster/gluster-object' | quote }}" +openshift_storage_glusterfs_s3_version: 'latest' +openshift_storage_glusterfs_s3_account: "{{ omit }}" +openshift_storage_glusterfs_s3_user: "{{ omit }}" +openshift_storage_glusterfs_s3_password: "{{ omit }}" +openshift_storage_glusterfs_s3_pvc: "gluster-s3-{{ openshift_storage_glusterfs_name }}-{{ openshift_storage_glusterfs_s3_account }}-claim" +openshift_storage_glusterfs_s3_pvc_size: "2Gi" +openshift_storage_glusterfs_s3_meta_pvc: "gluster-s3-{{ openshift_storage_glusterfs_name }}-{{ openshift_storage_glusterfs_s3_account }}-meta-claim" +openshift_storage_glusterfs_s3_meta_pvc_size: "1Gi" openshift_storage_glusterfs_wipe: False openshift_storage_glusterfs_heketi_is_native: "{{ openshift_storage_glusterfs_is_native }}" openshift_storage_glusterfs_heketi_is_missing: True @@ -25,6 +39,7 @@ openshift_storage_glusterfs_heketi_ssh_port: 22 openshift_storage_glusterfs_heketi_ssh_user: 'root' openshift_storage_glusterfs_heketi_ssh_sudo: False openshift_storage_glusterfs_heketi_ssh_keyfile: "{{ omit }}" +openshift_storage_glusterfs_heketi_fstab: "{{ '/var/lib/heketi/fstab' | quote if openshift_storage_glusterfs_heketi_executor == 'kubernetes' else '/etc/fstab' | quote }}" openshift_storage_glusterfs_namespace: "{{ 'glusterfs' | quote if openshift_storage_glusterfs_is_native or openshift_storage_glusterfs_heketi_is_native else 'default' | quote }}" openshift_storage_glusterfs_registry_timeout: "{{ openshift_storage_glusterfs_timeout }}" @@ -36,6 +51,20 @@ openshift_storage_glusterfs_registry_use_default_selector: "{{ openshift_storage openshift_storage_glusterfs_registry_storageclass: False openshift_storage_glusterfs_registry_image: "{{ openshift_storage_glusterfs_image }}" openshift_storage_glusterfs_registry_version: "{{ openshift_storage_glusterfs_version }}" +openshift_storage_glusterfs_registry_block_deploy: "{{ openshift_storage_glusterfs_block_deploy }}" +openshift_storage_glusterfs_registry_block_image: "{{ openshift_storage_glusterfs_block_image }}" +openshift_storage_glusterfs_registry_block_version: "{{ openshift_storage_glusterfs_block_version }}" +openshift_storage_glusterfs_registry_block_max_host_vol: "{{ openshift_storage_glusterfs_block_max_host_vol }}" +openshift_storage_glusterfs_registry_s3_deploy: "{{ openshift_storage_glusterfs_s3_deploy }}" +openshift_storage_glusterfs_registry_s3_image: "{{ openshift_storage_glusterfs_s3_image }}" +openshift_storage_glusterfs_registry_s3_version: "{{ openshift_storage_glusterfs_s3_version }}" +openshift_storage_glusterfs_registry_s3_account: "{{ openshift_storage_glusterfs_s3_account }}" +openshift_storage_glusterfs_registry_s3_user: "{{ openshift_storage_glusterfs_s3_user }}" +openshift_storage_glusterfs_registry_s3_password: "{{ openshift_storage_glusterfs_s3_password }}" +openshift_storage_glusterfs_registry_s3_pvc: "gluster-s3-{{ openshift_storage_glusterfs_registry_name }}-{{ openshift_storage_glusterfs_registry_s3_account }}-claim" +openshift_storage_glusterfs_registry_s3_pvc_size: "{{ openshift_storage_glusterfs_s3_pvc_size }}" +openshift_storage_glusterfs_registry_s3_meta_pvc: "gluster-s3-{{ openshift_storage_glusterfs_registry_name }}-{{ openshift_storage_glusterfs_registry_s3_account }}-meta-claim" +openshift_storage_glusterfs_registry_s3_meta_pvc_size: "{{ openshift_storage_glusterfs_s3_meta_pvc_size }}" openshift_storage_glusterfs_registry_wipe: "{{ openshift_storage_glusterfs_wipe }}" openshift_storage_glusterfs_registry_heketi_is_native: "{{ openshift_storage_glusterfs_registry_is_native }}" openshift_storage_glusterfs_registry_heketi_is_missing: "{{ openshift_storage_glusterfs_heketi_is_missing }}" @@ -54,6 +83,8 @@ openshift_storage_glusterfs_registry_heketi_ssh_port: "{{ openshift_storage_glus openshift_storage_glusterfs_registry_heketi_ssh_user: "{{ openshift_storage_glusterfs_heketi_ssh_user }}" openshift_storage_glusterfs_registry_heketi_ssh_sudo: "{{ openshift_storage_glusterfs_heketi_ssh_sudo }}" openshift_storage_glusterfs_registry_heketi_ssh_keyfile: "{{ openshift_storage_glusterfs_heketi_ssh_keyfile | default(omit) }}" +openshift_storage_glusterfs_registry_heketi_fstab: "{{ '/var/lib/heketi/fstab' | quote if openshift_storage_glusterfs_registry_heketi_executor == 'kubernetes' else '/etc/fstab' | quote }}" + r_openshift_storage_glusterfs_firewall_enabled: "{{ os_firewall_enabled | default(True) }}" r_openshift_storage_glusterfs_use_firewalld: "{{ os_firewall_use_firewalld | default(False) }}" r_openshift_storage_glusterfs_os_firewall_deny: [] diff --git a/roles/openshift_storage_glusterfs/files/v3.7/deploy-heketi-template.yml b/roles/openshift_storage_glusterfs/files/v3.7/deploy-heketi-template.yml index 7b705c2d4..34af652c2 100644 --- a/roles/openshift_storage_glusterfs/files/v3.7/deploy-heketi-template.yml +++ b/roles/openshift_storage_glusterfs/files/v3.7/deploy-heketi-template.yml @@ -73,13 +73,11 @@ objects: - name: HEKETI_EXECUTOR value: ${HEKETI_EXECUTOR} - name: HEKETI_FSTAB - value: /var/lib/heketi/fstab + value: ${HEKETI_FSTAB} - name: HEKETI_SNAPSHOT_LIMIT value: '14' - name: HEKETI_KUBE_GLUSTER_DAEMONSET value: '1' - - name: HEKETI_KUBE_NAMESPACE - value: ${HEKETI_KUBE_NAMESPACE} ports: - containerPort: 8080 volumeMounts: @@ -115,10 +113,10 @@ parameters: displayName: heketi executor type description: Set the executor type, kubernetes or ssh value: kubernetes -- name: HEKETI_KUBE_NAMESPACE - displayName: Namespace - description: Set the namespace where the GlusterFS pods reside - value: default +- name: HEKETI_FSTAB + displayName: heketi fstab path + description: Set the fstab path, file that is populated with bricks that heketi creates + value: /var/lib/heketi/fstab - name: HEKETI_ROUTE displayName: heketi route name description: Set the hostname for the route URL diff --git a/roles/openshift_storage_glusterfs/files/v3.7/gluster-s3-pvcs-template.yml b/roles/openshift_storage_glusterfs/files/v3.7/gluster-s3-pvcs-template.yml new file mode 100644 index 000000000..064b51473 --- /dev/null +++ b/roles/openshift_storage_glusterfs/files/v3.7/gluster-s3-pvcs-template.yml @@ -0,0 +1,67 @@ +--- +kind: Template +apiVersion: v1 +metadata: + name: gluster-s3-pvcs + labels: + glusterfs: s3-pvcs-template + gluster-s3: pvcs-template + annotations: + description: Gluster S3 service template + tags: glusterfs,heketi,gluster-s3 +objects: +- kind: PersistentVolumeClaim + apiVersion: v1 + metadata: + name: "${PVC}" + labels: + glusterfs: s3-${CLUSTER_NAME}-${S3_ACCOUNT}-storage + gluster-s3: ${CLUSTER_NAME}-${S3_ACCOUNT}-pvc + annotations: + volume.beta.kubernetes.io/storage-class: "glusterfs-${CLUSTER_NAME}" + spec: + accessModes: + - ReadWriteMany + resources: + requests: + storage: "${PVC_SIZE}" +- kind: PersistentVolumeClaim + apiVersion: v1 + metadata: + name: "${META_PVC}" + labels: + glusterfs: s3-${CLUSTER_NAME}-${S3_ACCOUNT}-storage + gluster-s3: ${CLUSTER_NAME}-${S3_ACCOUNT}-meta-pvc + annotations: + volume.beta.kubernetes.io/storage-class: "glusterfs-${CLUSTER_NAME}" + spec: + accessModes: + - ReadWriteMany + resources: + requests: + storage: "${META_PVC_SIZE}" +parameters: +- name: S3_ACCOUNT + displayName: S3 Account Name + description: S3 storage account which will provide storage on GlusterFS volumes + required: true +- name: PVC + displayName: Primary GlusterFS-backed PVC + description: GlusterFS-backed PVC for object storage + required: true +- name: PVC_SIZE + displayName: Primary GlusterFS-backed PVC capacity + description: Capacity for GlusterFS-backed PVC for object storage + value: 2Gi +- name: META_PVC + displayName: Metadata GlusterFS-backed PVC + description: GlusterFS-backed PVC for object storage metadata + required: true +- name: META_PVC_SIZE + displayName: Metadata GlusterFS-backed PVC capacity + description: Capacity for GlusterFS-backed PVC for object storage metadata + value: 1Gi +- name: CLUSTER_NAME + displayName: GlusterFS cluster name + description: A unique name to identify which heketi service manages this cluster, useful for running multiple heketi instances + value: storage diff --git a/roles/openshift_storage_glusterfs/files/v3.7/gluster-s3-template.yml b/roles/openshift_storage_glusterfs/files/v3.7/gluster-s3-template.yml new file mode 100644 index 000000000..896a1b226 --- /dev/null +++ b/roles/openshift_storage_glusterfs/files/v3.7/gluster-s3-template.yml @@ -0,0 +1,140 @@ +--- +kind: Template +apiVersion: v1 +metadata: + name: gluster-s3 + labels: + glusterfs: s3-template + gluster-s3: template + annotations: + description: Gluster S3 service template + tags: glusterfs,heketi,gluster-s3 +objects: +- kind: Service + apiVersion: v1 + metadata: + name: gluster-s3-${CLUSTER_NAME}-${S3_ACCOUNT}-service + labels: + glusterfs: s3-${CLUSTER_NAME}-${S3_ACCOUNT}-service + gluster-s3: ${CLUSTER_NAME}-${S3_ACCOUNT}-service + spec: + ports: + - protocol: TCP + port: 8080 + targetPort: 8080 + selector: + glusterfs: s3-pod + type: ClusterIP + sessionAffinity: None + status: + loadBalancer: {} +- kind: Route + apiVersion: v1 + metadata: + name: gluster-s3-${CLUSTER_NAME}-${S3_ACCOUNT}-route + labels: + glusterfs: s3-${CLUSTER_NAME}-${S3_ACCOUNT}-route + gluster-s3: ${CLUSTER_NAME}-${S3_ACCOUNT}-route + spec: + to: + kind: Service + name: gluster-s3-${CLUSTER_NAME}-${S3_ACCOUNT}-service +- kind: DeploymentConfig + apiVersion: v1 + metadata: + name: gluster-s3-${CLUSTER_NAME}-${S3_ACCOUNT}-dc + labels: + glusterfs: s3-${CLUSTER_NAME}-${S3_ACCOUNT}-dc + gluster-s3: ${CLUSTER_NAME}-${S3_ACCOUNT}-dc + annotations: + openshift.io/scc: privileged + description: Defines how to deploy gluster s3 object storage + spec: + replicas: 1 + selector: + glusterfs: s3-${CLUSTER_NAME}-${S3_ACCOUNT}-pod + template: + metadata: + name: gluster-${CLUSTER_NAME}-${S3_ACCOUNT}-s3 + labels: + glusterfs: s3-${CLUSTER_NAME}-${S3_ACCOUNT}-pod + gluster-s3: ${CLUSTER_NAME}-${S3_ACCOUNT}-pod + spec: + containers: + - name: gluster-s3 + image: ${IMAGE_NAME}:${IMAGE_VERSION} + imagePullPolicy: IfNotPresent + ports: + - name: gluster + containerPort: 8080 + protocol: TCP + env: + - name: S3_ACCOUNT + value: "${S3_ACCOUNT}" + - name: S3_USER + value: "${S3_USER}" + - name: S3_PASSWORD + value: "${S3_PASSWORD}" + resources: {} + volumeMounts: + - name: gluster-vol1 + mountPath: "/mnt/gluster-object/${S3_ACCOUNT}" + - name: gluster-vol2 + mountPath: "/mnt/gluster-object/gsmetadata" + - name: glusterfs-cgroup + readOnly: true + mountPath: "/sys/fs/cgroup" + terminationMessagePath: "/dev/termination-log" + securityContext: + privileged: true + volumes: + - name: glusterfs-cgroup + hostPath: + path: "/sys/fs/cgroup" + - name: gluster-vol1 + persistentVolumeClaim: + claimName: ${PVC} + - name: gluster-vol2 + persistentVolumeClaim: + claimName: ${META_PVC} + restartPolicy: Always + terminationGracePeriodSeconds: 30 + dnsPolicy: ClusterFirst + serviceAccountName: default + serviceAccount: default + securityContext: {} +parameters: +- name: IMAGE_NAME + displayName: glusterblock provisioner container image name + required: True +- name: IMAGE_VERSION + displayName: glusterblock provisioner container image version + required: True +- name: CLUSTER_NAME + displayName: GlusterFS cluster name + description: A unique name to identify which heketi service manages this cluster, useful for running multiple heketi instances + value: storage +- name: S3_ACCOUNT + displayName: S3 Account Name + description: S3 storage account which will provide storage on GlusterFS volumes + required: true +- name: S3_USER + displayName: S3 User + description: S3 user who can access the S3 storage account + required: true +- name: S3_PASSWORD + displayName: S3 User Password + description: Password for the S3 user + required: true +- name: PVC + displayName: Primary GlusterFS-backed PVC + description: GlusterFS-backed PVC for object storage + value: gluster-s3-claim +- name: META_PVC + displayName: Metadata GlusterFS-backed PVC + description: GlusterFS-backed PVC for object storage metadata + value: gluster-s3-meta-claim +- name: CLUSTER_NAME + displayName: GlusterFS cluster name + description: A unique name to identify which heketi service manages this cluster, useful for running multiple heketi instances + value: storage diff --git a/roles/openshift_storage_glusterfs/files/v3.7/glusterblock-provisioner.yml b/roles/openshift_storage_glusterfs/files/v3.7/glusterblock-provisioner.yml new file mode 100644 index 000000000..2cc69644c --- /dev/null +++ b/roles/openshift_storage_glusterfs/files/v3.7/glusterblock-provisioner.yml @@ -0,0 +1,105 @@ +--- +kind: Template +apiVersion: v1 +metadata: + name: glusterblock + labels: + glusterfs: block-template + glusterblock: template + annotations: + description: glusterblock provisioner template + tags: glusterfs +objects: +- kind: ClusterRole + apiVersion: v1 + metadata: + name: glusterblock-provisioner-runner + labels: + glusterfs: block-provisioner-runner-clusterrole + glusterblock: provisioner-runner-clusterrole + rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "create", "delete"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] + - apiGroups: [""] + resources: ["services"] + verbs: ["get"] + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "create", "delete"] + - apiGroups: [""] + resources: ["routes"] + verbs: ["get", "list"] +- apiVersion: v1 + kind: ServiceAccount + metadata: + name: glusterblock-${CLUSTER_NAME}-provisioner + labels: + glusterfs: block-${CLUSTER_NAME}-provisioner-sa + glusterblock: ${CLUSTER_NAME}-provisioner-sa +- apiVersion: v1 + kind: ClusterRoleBinding + metadata: + name: glusterblock-${CLUSTER_NAME}-provisioner + roleRef: + name: glusterblock-provisioner-runner + subjects: + - kind: ServiceAccount + name: glusterblock-${CLUSTER_NAME}-provisioner + namespace: ${NAMESPACE} +- kind: DeploymentConfig + apiVersion: v1 + metadata: + name: glusterblock-${CLUSTER_NAME}-provisioner-dc + labels: + glusterfs: block-${CLUSTER_NAME}-provisioner-dc + glusterblock: ${CLUSTER_NAME}-provisioner-dc + annotations: + description: Defines how to deploy the glusterblock provisioner pod. + spec: + replicas: 1 + selector: + glusterfs: block-${CLUSTER_NAME}-provisioner-pod + triggers: + - type: ConfigChange + strategy: + type: Recreate + template: + metadata: + name: glusterblock-provisioner + labels: + glusterfs: block-${CLUSTER_NAME}-provisioner-pod + spec: + serviceAccountName: glusterblock-${CLUSTER_NAME}-provisioner + containers: + - name: glusterblock-provisioner + image: ${IMAGE_NAME}:${IMAGE_VERSION} + image: gluster/glusterblock-provisioner:latest + imagePullPolicy: IfNotPresent + env: + - name: PROVISIONER_NAME + value: gluster.org/glusterblock +parameters: +- name: IMAGE_NAME + displayName: glusterblock provisioner container image name + required: True +- name: IMAGE_VERSION + displayName: glusterblock provisioner container image version + required: True +- name: NAMESPACE + displayName: glusterblock provisioner namespace + description: The namespace in which these resources are being created + required: True +- name: CLUSTER_NAME + displayName: GlusterFS cluster name + description: A unique name to identify which heketi service manages this cluster, useful for running multiple heketi instances + value: storage diff --git a/roles/openshift_storage_glusterfs/files/v3.7/glusterfs-template.yml b/roles/openshift_storage_glusterfs/files/v3.7/glusterfs-template.yml index 8c5e1ded3..09850a2c2 100644 --- a/roles/openshift_storage_glusterfs/files/v3.7/glusterfs-template.yml +++ b/roles/openshift_storage_glusterfs/files/v3.7/glusterfs-template.yml @@ -35,6 +35,15 @@ objects: - name: glusterfs image: ${IMAGE_NAME}:${IMAGE_VERSION} imagePullPolicy: IfNotPresent + env: + - name: GB_GLFS_LRU_COUNT + value: "${GB_GLFS_LRU_COUNT}" + - name: TCMU_LOGDIR + value: "${TCMU_LOGDIR}" + resources: + requests: + memory: 100Mi + cpu: 100m volumeMounts: - name: glusterfs-heketi mountPath: "/var/lib/heketi" @@ -83,7 +92,6 @@ objects: periodSeconds: 25 successThreshold: 1 failureThreshold: 15 - resources: {} terminationMessagePath: "/dev/termination-log" volumes: - name: glusterfs-heketi @@ -134,3 +142,13 @@ parameters: displayName: GlusterFS cluster name description: A unique name to identify which heketi service manages this cluster, useful for running multiple heketi instances value: storage +- name: GB_GLFS_LRU_COUNT + displayName: Maximum number of block hosting volumes + description: This value is to set maximum number of block hosting volumes. + value: "15" + required: true +- name: TCMU_LOGDIR + displayName: Tcmu runner log directory + description: This value is to set tcmu runner log directory + value: "/var/log/glusterfs/gluster-block" + required: true diff --git a/roles/openshift_storage_glusterfs/files/v3.7/heketi-template.yml b/roles/openshift_storage_glusterfs/files/v3.7/heketi-template.yml index 61b6a8c13..28cdb2982 100644 --- a/roles/openshift_storage_glusterfs/files/v3.7/heketi-template.yml +++ b/roles/openshift_storage_glusterfs/files/v3.7/heketi-template.yml @@ -15,6 +15,7 @@ objects: name: heketi-${CLUSTER_NAME} labels: glusterfs: heketi-${CLUSTER_NAME}-service + heketi: ${CLUSTER_NAME}-service annotations: description: Exposes Heketi service spec: @@ -30,6 +31,7 @@ objects: name: ${HEKETI_ROUTE} labels: glusterfs: heketi-${CLUSTER_NAME}-route + heketi: ${CLUSTER_NAME}-route spec: to: kind: Service @@ -40,6 +42,7 @@ objects: name: heketi-${CLUSTER_NAME} labels: glusterfs: heketi-${CLUSTER_NAME}-dc + heketi: ${CLUSTER_NAME}-dc annotations: description: Defines how to deploy Heketi spec: @@ -55,6 +58,7 @@ objects: name: heketi-${CLUSTER_NAME} labels: glusterfs: heketi-${CLUSTER_NAME}-pod + heketi: ${CLUSTER_NAME}-pod spec: serviceAccountName: heketi-${CLUSTER_NAME}-service-account containers: @@ -69,13 +73,11 @@ objects: - name: HEKETI_EXECUTOR value: ${HEKETI_EXECUTOR} - name: HEKETI_FSTAB - value: /var/lib/heketi/fstab + value: ${HEKETI_FSTAB} - name: HEKETI_SNAPSHOT_LIMIT value: '14' - name: HEKETI_KUBE_GLUSTER_DAEMONSET value: '1' - - name: HEKETI_KUBE_NAMESPACE - value: ${HEKETI_KUBE_NAMESPACE} ports: - containerPort: 8080 volumeMounts: @@ -114,10 +116,10 @@ parameters: displayName: heketi executor type description: Set the executor type, kubernetes or ssh value: kubernetes -- name: HEKETI_KUBE_NAMESPACE - displayName: Namespace - description: Set the namespace where the GlusterFS pods reside - value: default +- name: HEKETI_FSTAB + displayName: heketi fstab path + description: Set the fstab path, file that is populated with bricks that heketi creates + value: /var/lib/heketi/fstab - name: HEKETI_ROUTE displayName: heketi route name description: Set the hostname for the route URL diff --git a/roles/openshift_storage_glusterfs/tasks/gluster_s3_deploy.yml b/roles/openshift_storage_glusterfs/tasks/gluster_s3_deploy.yml new file mode 100644 index 000000000..1664ecc1e --- /dev/null +++ b/roles/openshift_storage_glusterfs/tasks/gluster_s3_deploy.yml @@ -0,0 +1,113 @@ +--- +- name: Delete pre-existing gluster-s3 resources + oc_obj: + namespace: "{{ glusterfs_namespace }}" + kind: "{{ item.kind }}" + name: "{{ item.name | default(omit) }}" + selector: "{{ item.selector | default(omit) }}" + state: absent + with_items: + - kind: "all,svc,deploy,secret,sc,pvc" + selector: "gluster-s3" + failed_when: False + when: glusterfs_wipe + +- name: Wait for gluster-s3 pods to terminate + oc_obj: + namespace: "{{ glusterfs_namespace }}" + kind: pod + state: list + selector: "glusterfs=s3-{{ glusterfs_name }}-provisioner-pod" + register: gluster_s3_pod + until: "gluster_s3_pod.results.results[0]['items'] | count == 0" + delay: 10 + retries: "{{ (glusterfs_timeout | int / 10) | int }}" + when: glusterfs_wipe + +- name: Copy gluster-s3 PVCs template file + copy: + src: "{{ openshift.common.examples_content_version }}/{{ item }}" + dest: "{{ mktemp.stdout }}/{{ item }}" + with_items: + - "gluster-s3-pvcs-template.yml" + +- name: Create gluster-s3 PVCs template + oc_obj: + namespace: "{{ glusterfs_namespace }}" + kind: template + name: "gluster-s3-pvcs" + state: present + files: + - "{{ mktemp.stdout }}/gluster-s3-pvcs-template.yml" + +- name: Create gluster-s3 PVCs + oc_process: + namespace: "{{ glusterfs_namespace }}" + template_name: "gluster-s3-pvcs" + create: True + params: + S3_ACCOUNT: "{{ glusterfs_s3_account }}" + PVC: "{{ glusterfs_s3_pvc }}" + PVC_SIZE: "{{ glusterfs_s3_pvc_size }}" + META_PVC: "{{ glusterfs_s3_meta_pvc }}" + META_PVC_SIZE: "{{ glusterfs_s3_meta_pvc_size }}" + CLUSTER_NAME: "{{ glusterfs_name }}" + +- name: Wait for gluster-s3 PVCs + oc_obj: + namespace: "{{ glusterfs_namespace }}" + kind: pvc + state: list + selector: "glusterfs=s3-{{ glusterfs_name }}-{{ glusterfs_s3_account }}-storage" + register: gluster_s3_pvcs + until: + - "gluster_s3_pvcs.results.results[0]['items'] | count > 0" + # Pod's 'Bound' status must be True + - "gluster_s3_pvcs.results.results[0]['items'] | oo_collect(attribute='status.conditions') | oo_collect(attribute='status', filters={'type': 'Bound'}) | map('bool') | select | list | count == 2" + delay: 10 + retries: "{{ (glusterfs_timeout | int / 10) | int }}" + +- name: Copy gluster-s3 template file + copy: + src: "{{ openshift.common.examples_content_version }}/{{ item }}" + dest: "{{ mktemp.stdout }}/{{ item }}" + with_items: + - "gluster-s3-template.yml" + +- name: Create gluster-s3 template + oc_obj: + namespace: "{{ glusterfs_namespace }}" + kind: template + name: "gluster-s3" + state: present + files: + - "{{ mktemp.stdout }}/gluster-s3-template.yml" + +- name: Deploy gluster-s3 service + oc_process: + namespace: "{{ glusterfs_namespace }}" + template_name: "gluster-s3" + create: True + params: + IMAGE_NAME: "{{ glusterfs_s3_image }}" + IMAGE_VERSION: "{{ glusterfs_s3_version }}" + S3_ACCOUNT: "{{ glusterfs_s3_account }}" + S3_USER: "{{ glusterfs_s3_user }}" + S3_PASSWORD: "{{ glusterfs_s3_password }}" + PVC: "{{ glusterfs_s3_pvc }}" + META_PVC: "{{ glusterfs_s3_meta_pvc }}" + CLUSTER_NAME: "{{ glusterfs_name }}" + +- name: Wait for gluster-s3 pod + oc_obj: + namespace: "{{ glusterfs_namespace }}" + kind: pod + state: list + selector: "glusterfs=s3-{{ glusterfs_name }}-{{ glusterfs_s3_account }}-pod" + register: gluster_s3_pod + until: + - "gluster_s3_pod.results.results[0]['items'] | count > 0" + # Pod's 'Ready' status must be True + - "gluster_s3_pod.results.results[0]['items'] | oo_collect(attribute='status.conditions') | oo_collect(attribute='status', filters={'type': 'Ready'}) | map('bool') | select | list | count == 1" + delay: 10 + retries: "{{ (glusterfs_timeout | int / 10) | int }}" diff --git a/roles/openshift_storage_glusterfs/tasks/glusterblock_deploy.yml b/roles/openshift_storage_glusterfs/tasks/glusterblock_deploy.yml new file mode 100644 index 000000000..bba1de654 --- /dev/null +++ b/roles/openshift_storage_glusterfs/tasks/glusterblock_deploy.yml @@ -0,0 +1,66 @@ +--- +- name: Delete pre-existing glusterblock provisioner resources + oc_obj: + namespace: "{{ glusterfs_namespace }}" + kind: "{{ item.kind }}" + name: "{{ item.name | default(omit) }}" + selector: "{{ item.selector | default(omit) }}" + state: absent + with_items: + - kind: "all,deploy,sa,clusterrole,clusterrolebinding" + selector: "glusterblock" + failed_when: False + when: glusterfs_wipe + +- name: Wait for glusterblock provisioner pods to terminate + oc_obj: + namespace: "{{ glusterfs_namespace }}" + kind: pod + state: list + selector: "glusterfs=block-{{ glusterfs_name }}-provisioner-pod" + register: glusterblock_pod + until: "glusterblock_pod.results.results[0]['items'] | count == 0" + delay: 10 + retries: "{{ (glusterfs_timeout | int / 10) | int }}" + when: glusterfs_wipe + +- name: Copy initial glusterblock provisioner resource file + copy: + src: "{{ openshift.common.examples_content_version }}/{{ item }}" + dest: "{{ mktemp.stdout }}/{{ item }}" + with_items: + - "glusterblock-template.yml" + +- name: Create glusterblock provisioner template + oc_obj: + namespace: "{{ glusterfs_namespace }}" + kind: template + name: "glusterblock" + state: present + files: + - "{{ mktemp.stdout }}/glusterblock-template.yml" + +- name: Deploy glusterblock provisioner + oc_process: + namespace: "{{ glusterfs_namespace }}" + template_name: "glusterblock" + create: True + params: + IMAGE_NAME: "{{ glusterfs_block_image }}" + IMAGE_VERSION: "{{ glusterfs_block_version }}" + NAMESPACE: "{{ glusterfs_namespace }}" + CLUSTER_NAME: "{{ glusterfs_name }}" + +- name: Wait for glusterblock provisioner pod + oc_obj: + namespace: "{{ glusterfs_namespace }}" + kind: pod + state: list + selector: "glusterfs=block-{{ glusterfs_name }}-provisioner-pod" + register: glusterblock_pod + until: + - "glusterblock_pod.results.results[0]['items'] | count > 0" + # Pod's 'Ready' status must be True + - "glusterblock_pod.results.results[0]['items'] | oo_collect(attribute='status.conditions') | oo_collect(attribute='status', filters={'type': 'Ready'}) | map('bool') | select | list | count == 1" + delay: 10 + retries: "{{ (glusterfs_timeout | int / 10) | int }}" diff --git a/roles/openshift_storage_glusterfs/tasks/glusterfs_common.yml b/roles/openshift_storage_glusterfs/tasks/glusterfs_common.yml index 51724f979..2a678af57 100644 --- a/roles/openshift_storage_glusterfs/tasks/glusterfs_common.yml +++ b/roles/openshift_storage_glusterfs/tasks/glusterfs_common.yml @@ -259,51 +259,59 @@ - glusterfs_heketi_is_native - glusterfs_heketi_is_missing -- name: Create heketi secret - oc_secret: - namespace: "{{ glusterfs_namespace }}" - state: present - name: "heketi-{{ glusterfs_name }}-admin-secret" - type: "kubernetes.io/glusterfs" - force: True - contents: - - path: key - data: "{{ glusterfs_heketi_admin_key }}" - when: - - glusterfs_storageclass - - glusterfs_heketi_admin_key is defined - -- name: Get heketi route - oc_obj: - namespace: "{{ glusterfs_namespace }}" - kind: route - state: list - name: "heketi-{{ glusterfs_name }}" - register: heketi_route - when: - - glusterfs_storageclass - - glusterfs_heketi_is_native - -- name: Determine StorageClass heketi URL +- name: Check if gluster-s3 can't be deployed set_fact: - glusterfs_heketi_route: "{{ heketi_route.results.results[0]['spec']['host'] }}" + glusterfs_s3_deploy: False when: - - glusterfs_storageclass - - glusterfs_heketi_is_native - -- name: Generate GlusterFS StorageClass file - template: - src: "{{ openshift.common.examples_content_version }}/glusterfs-storageclass.yml.j2" - dest: "{{ mktemp.stdout }}/glusterfs-storageclass.yml" + - "glusterfs_s3_account is not defined or glusterfs_s3_user is not defined or glusterfs_s3_password is not defined" + +- block: + - name: Create heketi secret + oc_secret: + namespace: "{{ glusterfs_namespace }}" + state: present + name: "heketi-{{ glusterfs_name }}-admin-secret" + type: "kubernetes.io/glusterfs" + force: True + contents: + - path: key + data: "{{ glusterfs_heketi_admin_key }}" + when: + - glusterfs_heketi_admin_key is defined + + - name: Get heketi route + oc_obj: + namespace: "{{ glusterfs_namespace }}" + kind: route + state: list + name: "heketi-{{ glusterfs_name }}" + register: heketi_route + when: + - glusterfs_heketi_is_native + + - name: Determine StorageClass heketi URL + set_fact: + glusterfs_heketi_route: "{{ heketi_route.results.results[0]['spec']['host'] }}" + when: + - glusterfs_heketi_is_native + + - name: Generate GlusterFS StorageClass file + template: + src: "{{ openshift.common.examples_content_version }}/glusterfs-storageclass.yml.j2" + dest: "{{ mktemp.stdout }}/glusterfs-storageclass.yml" + + - name: Create GlusterFS StorageClass + oc_obj: + state: present + kind: storageclass + name: "glusterfs-{{ glusterfs_name }}" + files: + - "{{ mktemp.stdout }}/glusterfs-storageclass.yml" when: - - glusterfs_storageclass + - glusterfs_storageclass or glusterfs_s3_deploy -- name: Create GlusterFS StorageClass - oc_obj: - state: present - kind: storageclass - name: "glusterfs-{{ glusterfs_name }}" - files: - - "{{ mktemp.stdout }}/glusterfs-storageclass.yml" - when: - - glusterfs_storageclass +- include: glusterblock_deploy.yml + when: glusterfs_block_deploy + +- include: gluster_s3_deploy.yml + when: glusterfs_s3_deploy diff --git a/roles/openshift_storage_glusterfs/tasks/glusterfs_config.yml b/roles/openshift_storage_glusterfs/tasks/glusterfs_config.yml index 012c722ff..e2d740f35 100644 --- a/roles/openshift_storage_glusterfs/tasks/glusterfs_config.yml +++ b/roles/openshift_storage_glusterfs/tasks/glusterfs_config.yml @@ -9,6 +9,20 @@ glusterfs_storageclass: "{{ openshift_storage_glusterfs_storageclass }}" glusterfs_image: "{{ openshift_storage_glusterfs_image }}" glusterfs_version: "{{ openshift_storage_glusterfs_version }}" + glusterfs_block_deploy: "{{ openshift_storage_glusterfs_block_deploy | bool }}" + glusterfs_block_image: "{{ openshift_storage_glusterfs_block_image }}" + glusterfs_block_version: "{{ openshift_storage_glusterfs_block_version }}" + glusterfs_block_max_host_vol: "{{ openshift_storage_glusterfs_block_max_host_vol }}" + glusterfs_s3_deploy: "{{ openshift_storage_glusterfs_s3_deploy | bool }}" + glusterfs_s3_image: "{{ openshift_storage_glusterfs_s3_image }}" + glusterfs_s3_version: "{{ openshift_storage_glusterfs_s3_version }}" + glusterfs_s3_account: "{{ openshift_storage_glusterfs_s3_account }}" + glusterfs_s3_user: "{{ openshift_storage_glusterfs_s3_user }}" + glusterfs_s3_password: "{{ openshift_storage_glusterfs_s3_password }}" + glusterfs_s3_pvc: "{{ openshift_storage_glusterfs_s3_pvc }}" + glusterfs_s3_pvc_size: "{{ openshift_storage_glusterfs_s3_pvc_size }}" + glusterfs_s3_meta_pvc: "{{ openshift_storage_glusterfs_s3_meta_pvc }}" + glusterfs_s3_meta_pvc_size: "{{ openshift_storage_glusterfs_s3_meta_pvc_size }}" glusterfs_wipe: "{{ openshift_storage_glusterfs_wipe | bool }}" glusterfs_heketi_is_native: "{{ openshift_storage_glusterfs_heketi_is_native | bool }}" glusterfs_heketi_is_missing: "{{ openshift_storage_glusterfs_heketi_is_missing | bool }}" @@ -27,6 +41,7 @@ glusterfs_heketi_ssh_user: "{{ openshift_storage_glusterfs_heketi_ssh_user }}" glusterfs_heketi_ssh_sudo: "{{ openshift_storage_glusterfs_heketi_ssh_sudo | bool }}" glusterfs_heketi_ssh_keyfile: "{{ openshift_storage_glusterfs_heketi_ssh_keyfile }}" + glusterfs_heketi_fstab: "{{ openshift_storage_glusterfs_heketi_fstab }}" glusterfs_nodes: "{{ groups.glusterfs }}" - include: glusterfs_common.yml diff --git a/roles/openshift_storage_glusterfs/tasks/glusterfs_deploy.yml b/roles/openshift_storage_glusterfs/tasks/glusterfs_deploy.yml index 932d06038..f98d4c62f 100644 --- a/roles/openshift_storage_glusterfs/tasks/glusterfs_deploy.yml +++ b/roles/openshift_storage_glusterfs/tasks/glusterfs_deploy.yml @@ -87,6 +87,7 @@ IMAGE_VERSION: "{{ glusterfs_version }}" NODE_LABELS: "{{ glusterfs_nodeselector }}" CLUSTER_NAME: "{{ glusterfs_name }}" + GB_GLFS_LRU_COUNT: "{{ glusterfs_block_max_host_vol }}" - name: Wait for GlusterFS pods oc_obj: diff --git a/roles/openshift_storage_glusterfs/tasks/glusterfs_registry.yml b/roles/openshift_storage_glusterfs/tasks/glusterfs_registry.yml index 1bcab8e49..baac52179 100644 --- a/roles/openshift_storage_glusterfs/tasks/glusterfs_registry.yml +++ b/roles/openshift_storage_glusterfs/tasks/glusterfs_registry.yml @@ -9,6 +9,20 @@ glusterfs_storageclass: "{{ openshift_storage_glusterfs_registry_storageclass }}" glusterfs_image: "{{ openshift_storage_glusterfs_registry_image }}" glusterfs_version: "{{ openshift_storage_glusterfs_registry_version }}" + glusterfs_block_deploy: "{{ openshift_storage_glusterfs_registry_block_deploy | bool }}" + glusterfs_block_image: "{{ openshift_storage_glusterfs_registry_block_image }}" + glusterfs_block_version: "{{ openshift_storage_glusterfs_registry_block_version }}" + glusterfs_block_max_host_vol: "{{ openshift_storage_glusterfs_registry_block_max_host_vol }}" + glusterfs_s3_deploy: "{{ openshift_storage_glusterfs_registry_s3_deploy | bool }}" + glusterfs_s3_image: "{{ openshift_storage_glusterfs_registry_s3_image }}" + glusterfs_s3_version: "{{ openshift_storage_glusterfs_registry_s3_version }}" + glusterfs_s3_account: "{{ openshift_storage_glusterfs_registry_s3_account }}" + glusterfs_s3_user: "{{ openshift_storage_glusterfs_registry_s3_user }}" + glusterfs_s3_password: "{{ openshift_storage_glusterfs_registry_s3_password }}" + glusterfs_s3_pvc: "{{ openshift_storage_glusterfs_registry_s3_pvc }}" + glusterfs_s3_pvc_size: "{{ openshift_storage_glusterfs_registry_s3_pvc_size }}" + glusterfs_s3_meta_pvc: "{{ openshift_storage_glusterfs_registry_s3_meta_pvc }}" + glusterfs_s3_meta_pvc_size: "{{ openshift_storage_glusterfs_registry_s3_meta_pvc_size }}" glusterfs_wipe: "{{ openshift_storage_glusterfs_registry_wipe | bool }}" glusterfs_heketi_is_native: "{{ openshift_storage_glusterfs_registry_heketi_is_native | bool }}" glusterfs_heketi_is_missing: "{{ openshift_storage_glusterfs_registry_heketi_is_missing | bool }}" @@ -27,6 +41,7 @@ glusterfs_heketi_ssh_user: "{{ openshift_storage_glusterfs_registry_heketi_ssh_user }}" glusterfs_heketi_ssh_sudo: "{{ openshift_storage_glusterfs_registry_heketi_ssh_sudo | bool }}" glusterfs_heketi_ssh_keyfile: "{{ openshift_storage_glusterfs_registry_heketi_ssh_keyfile }}" + glusterfs_heketi_fstab: "{{ openshift_storage_glusterfs_registry_heketi_fstab }}" glusterfs_nodes: "{{ groups.glusterfs_registry | default(groups.glusterfs) }}" - include: glusterfs_common.yml diff --git a/roles/openshift_storage_glusterfs/tasks/heketi_deploy_part1.yml b/roles/openshift_storage_glusterfs/tasks/heketi_deploy_part1.yml index 73396c9af..935d3b689 100644 --- a/roles/openshift_storage_glusterfs/tasks/heketi_deploy_part1.yml +++ b/roles/openshift_storage_glusterfs/tasks/heketi_deploy_part1.yml @@ -27,7 +27,7 @@ HEKETI_USER_KEY: "{{ glusterfs_heketi_user_key }}" HEKETI_ADMIN_KEY: "{{ glusterfs_heketi_admin_key }}" HEKETI_EXECUTOR: "{{ glusterfs_heketi_executor }}" - HEKETI_KUBE_NAMESPACE: "{{ glusterfs_namespace }}" + HEKETI_FSTAB: "{{ glusterfs_heketi_fstab }}" CLUSTER_NAME: "{{ glusterfs_name }}" - name: Set heketi Deployed fact diff --git a/roles/openshift_storage_glusterfs/tasks/heketi_deploy_part2.yml b/roles/openshift_storage_glusterfs/tasks/heketi_deploy_part2.yml index 54a6dd7c3..d23bd42b9 100644 --- a/roles/openshift_storage_glusterfs/tasks/heketi_deploy_part2.yml +++ b/roles/openshift_storage_glusterfs/tasks/heketi_deploy_part2.yml @@ -107,7 +107,7 @@ HEKETI_USER_KEY: "{{ glusterfs_heketi_user_key }}" HEKETI_ADMIN_KEY: "{{ glusterfs_heketi_admin_key }}" HEKETI_EXECUTOR: "{{ glusterfs_heketi_executor }}" - HEKETI_KUBE_NAMESPACE: "{{ glusterfs_namespace }}" + HEKETI_FSTAB: "{{ glusterfs_heketi_fstab }}" CLUSTER_NAME: "{{ glusterfs_name }}" - name: Wait for heketi pod |