reneable healthcheck for csi-node-driver

* fix snapshotter port
* update snapshotter rbac

.gitignore

@@ -0,0 +1,3 @@
+bin/
+config/client-info.yml
+test/sanity/sanity-test-secret-file.yaml

Dockerfile

@@ -1,26 +1,31 @@
 # Copyright 2021 Synology Inc.
 
 ############## Build stage ##############
-FROM golang:1.13.6-alpine as builder
+FROM golang:1.16.8-alpine as builder
 LABEL stage=synobuilder
 
 RUN apk add --no-cache alpine-sdk
 WORKDIR /go/src/synok8scsiplugin
 COPY go.mod .
+COPY go.sum .
 RUN go mod download
 
 COPY Makefile .
 
+ARG TARGETPLATFORM
+
 COPY main.go .
 COPY pkg ./pkg
-RUN make
+RUN env GOARCH=$(echo "$TARGETPLATFORM" | cut -f2 -d/) \
+        GOARM=$(echo "$TARGETPLATFORM" | cut -f3 -d/ | cut -c2-) \
+        make
 
 ############## Final stage ##############
 FROM alpine:latest
 LABEL maintainers="Synology Authors" \
       description="Synology CSI Plugin"
 
-RUN apk add --no-cache e2fsprogs e2fsprogs-extra xfsprogs xfsprogs-extra blkid util-linux iproute2 bash
+RUN apk add --no-cache e2fsprogs e2fsprogs-extra xfsprogs xfsprogs-extra blkid util-linux iproute2 bash btrfs-progs ca-certificates cifs-utils
 
 # Create symbolic link for chroot.sh
 WORKDIR /

Makefile

@@ -2,12 +2,15 @@
 
 REGISTRY_NAME=synology
 IMAGE_NAME=synology-csi
-IMAGE_VERSION=v1.0.0
+IMAGE_VERSION=v1.1.0
 IMAGE_TAG=$(REGISTRY_NAME)/$(IMAGE_NAME):$(IMAGE_VERSION)
 
 # For now, only build linux/amd64 platform
-GOARCH?=amd64
-BUILD_ENV=CGO_ENABLED=0 GOOS=linux GOARCH=$(GOARCH)
+ifeq ($(GOARCH),)
+GOARCH:=amd64
+endif
+GOARM?=""
+BUILD_ENV=CGO_ENABLED=0 GOOS=linux GOARCH=$(GOARCH) GOARM=$(GOARM)
 BUILD_FLAGS="-extldflags \"-static\""
 
 .PHONY: all clean synology-csi-driver synocli test docker-build
@@ -21,11 +24,15 @@ synology-csi-driver:
 docker-build:
 	docker build -f Dockerfile -t $(IMAGE_TAG) .
 
+docker-build-multiarch:
+	docker buildx build -t $(IMAGE_TAG) --platform linux/amd64,linux/arm/v7,linux/arm64 . --push
+
 synocli:
 	@mkdir -p bin
 	$(BUILD_ENV) go build -v -ldflags $(BUILD_FLAGS) -o ./bin/synocli ./synocli
 
 test:
+	go clean -testcache
 	go test -v ./test/...
 clean:
 	-rm -rf ./bin

README.md

@@ -4,9 +4,11 @@ The official [Container Storage Interface](https://github.com/container-storage-
 
 ### Container Images & Kubernetes Compatibility
 Driver Name: csi.san.synology.com
-| Driver Version | Image                                                                 | Supported K8s Version |
-| -------------- | --------------------------------------------------------------------- | --------------------- |
-| v1.0.0         | [synology-csi:v1.0.0](https://hub.docker.com/r/synology/synology-csi) | 1.19                  |
+| Driver Version                                                                   | Image                                                                 | Supported K8s Version |
+| -------------------------------------------------------------------------------- | --------------------------------------------------------------------- | --------------------- |
+| [v1.1.0](https://github.com/SynologyOpenSource/synology-csi/tree/release-v1.1.0) | [synology-csi:v1.1.0](https://hub.docker.com/r/synology/synology-csi) | 1.20+                 |
+| [v1.0.1](https://github.com/SynologyOpenSource/synology-csi/tree/release-v1.0.1) | [synology-csi:v1.0.1](https://hub.docker.com/r/synology/synology-csi) | 1.20+                 |
+| [v1.0.0](https://github.com/SynologyOpenSource/synology-csi/tree/release-v1.0.0) | [synology-csi:v1.0.0](https://hub.docker.com/r/synology/synology-csi) | 1.19                  |
 
 
 
@@ -18,7 +20,7 @@ The Synology CSI driver supports:
 
 ## Installation
 ### Prerequisites
-- Kubernetes versions 1.19
+- Kubernetes versions 1.19 or above
 - Synology NAS running DSM 7.0 or above
 - Go version 1.16 or above is recommended
 - (Optional) Both [Volume Snapshot CRDs](https://github.com/kubernetes-csi/external-snapshotter/tree/v4.0.0/client/config/crd) and the [common snapshot controller](https://github.com/kubernetes-csi/external-snapshotter/tree/v4.0.0/deploy/kubernetes/snapshot-controller) must be installed in your Kubernetes cluster if you want to use the **Snapshot** feature
@@ -98,6 +100,7 @@ Create and apply StorageClasses with the properties you want.
 
 1. Create YAML files using the one at `deploy/kubernetes/<k8s version>/storage-class.yml` as the example, whose content is as below:
 
+    **iSCSI Protocol**
     ```
     apiVersion: storage.k8s.io/v1
     kind: StorageClass
@@ -113,18 +116,56 @@ Create and apply StorageClasses with the properties you want.
     reclaimPolicy: Retain
     allowVolumeExpansion: true
     ```
+
+    **SMB/CIFS Protocol**
+
+    Before creating an SMB/CIFS storage class, you must **create a secret** and specify the DSM user whom you want to give permissions to.
+
+    ```
+    apiVersion: v1
+    kind: Secret
+    metadata:
+      name: cifs-csi-credentials
+      namespace: default
+    type: Opaque
+    stringData:
+      username: <username>  # DSM user account accessing the shared folder
+      password: <password>  # DSM user password accessing the shared folder
+    ```
+
+    After creating the secret, create a storage class and fill the secret for node-stage-secret. This is a **required** step if you're using SMB, or there will be errors when staging volumes.
+
+    ```
+    apiVersion: storage.k8s.io/v1
+    kind: StorageClass
+    metadata:
+      name: synostorage-smb
+    provisioner: csi.san.synology.com
+    parameters:
+      protocol: "smb"
+      dsm: '192.168.1.1'
+      location: '/volume1'
+      csi.storage.k8s.io/node-stage-secret-name: "cifs-csi-credentials"
+      csi.storage.k8s.io/node-stage-secret-namespace: "default"
+    reclaimPolicy: Delete
+    allowVolumeExpansion: true
+    ```
+
 2. Configure the StorageClass properties by assigning the parameters in the table. You can also leave blank if you don’t have a preference:
 
-    | Name       | Type   | Description                                                                                                       | Default |
-    | ---------- | ------ | ----------------------------------------------------------------------------------------------------------------- | ------- |
-    | *dsm*      | string | The IPv4 address of your DSM, which must be included in the `client-info.yml` for the CSI driver to log in to DSM | -       |
-    | *location* | string | The location (/volume1, /volume2, ...) on DSM where the LUN for *PersistentVolume* will be created                | -       |
-    | *fsType*   | string | The formatting file system of the *PersistentVolumes* when you mount them on the pods                             | 'ext4'  |
+    | Name                                             | Type   | Description                                                                                                                                                        | Default | Supported protocols |
+    | ------------------------------------------------ | ------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ------- | ------------------- |
+    | *dsm*                                            | string | The IPv4 address of your DSM, which must be included in the `client-info.yml` for the CSI driver to log in to DSM                                                  | -       | iSCSI, SMB          |
+    | *location*                                       | string | The location (/volume1, /volume2, ...) on DSM where the LUN for *PersistentVolume* will be created                                                                 | -       | iSCSI, SMB          |
+    | *fsType*                                         | string | The formatting file system of the *PersistentVolumes* when you mount them on the pods. This parameter only works with iSCSI. For SMB, the fsType is always ‘cifs‘. | 'ext4'  | iSCSI               |
+    | *protocol*                                       | string | The backing storage protocol. Enter ‘iscsi’ to create LUNs or ‘smb‘ to create shared folders on DSM.                                                               | 'iscsi' | iSCSI, SMB          |
+    | *csi.storage.k8s.io/node-stage-secret-name*      | string | The name of node-stage-secret. Required if DSM shared folder is accessed via SMB.                                                                                  | -       | SMB                 |
+    | *csi.storage.k8s.io/node-stage-secret-namespace* | string | The namespace of node-stage-secret. Required if DSM shared folder is accessed via SMB.                                                                             | -       | SMB                 |
 
     **Notice**
 
     - If you leave the parameter *location* blank, the CSI driver will choose a volume on DSM with available storage to create the volumes.
-    - All volumes created by the CSI driver are Thin Provisioned LUNs on DSM. This will allow you to take snapshots of them.
+    - All iSCSI volumes created by the CSI driver are Thin Provisioned LUNs on DSM. This will allow you to take snapshots of them.
 
 3. Apply the YAML files to the Kubernetes cluster.
 
@@ -153,10 +194,10 @@ Create and apply VolumeSnapshotClasses with the properties you want.
 
 2. Configure volume snapshot class properties by assigning the following parameters, all parameters are optional:
 
-    | Name          | Type   | Description                                  | Default |
-    | ------------- | ------ | -------------------------------------------- | ------- |
-    | *description* | string | The description of the snapshot on DSM       | ""      |
-    | *is_locked*   | string | Whether you want to lock the snapshot on DSM | 'false' |
+    | Name          | Type   | Description                                  | Default | Supported protocols |
+    | ------------- | ------ | -------------------------------------------- | ------- | ------------------- |
+    | *description* | string | The description of the snapshot on DSM       | ""      | iSCSI               |
+    | *is_locked*   | string | Whether you want to lock the snapshot on DSM | 'false' | iSCSI, SMB          |
 
 3. Apply the YAML files to the Kubernetes cluster.
 

deploy/example/storageclass-smb.yaml

@@ -0,0 +1,20 @@
+---
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: synology-smb-storage
+provisioner: csi.san.synology.com
+parameters:
+  protocol: "smb" # required for smb protocol
+  # Before creating an SMB storage class, you must create a secret and specify the DSM user whom you want to give permissions to.
+  csi.storage.k8s.io/node-stage-secret-name: "cifs-csi-credentials" # required for smb protocol
+  csi.storage.k8s.io/node-stage-secret-namespace: "default"         # required for smb protocol
+  # dsm: "1.1.1.1"
+  # location: '/volume1'
+# mountOptions:
+#   - dir_mode=0777
+#   - file_mode=0777
+#   - uid=0
+#   - gid=0
+reclaimPolicy: Delete
+allowVolumeExpansion: true

deploy/kubernetes/v1.19/controller.yml

@@ -25,6 +25,9 @@ rules:
   - apiGroups: [""]
     resources: ["nodes"]
     verbs: ["get", "list", "watch"]
+  - apiGroups: [""]
+    resources: ["pods"]
+    verbs: ["get", "list", "watch"]
   - apiGroups: ["storage.k8s.io"]
     resources: ["csinodes"]
     verbs: ["get", "list", "watch"]
@@ -32,7 +35,7 @@ rules:
     resources: ["csinodeinfos"]
     verbs: ["get", "list", "watch"]
   - apiGroups: ["storage.k8s.io"]
-    resources: ["volumeattachments"]
+    resources: ["volumeattachments", "volumeattachments/status"]
     verbs: ["get", "list", "watch", "update", "patch"]
   - apiGroups: ["storage.k8s.io"]
     resources: ["storageclasses"]
@@ -43,6 +46,9 @@ rules:
   - apiGroups: ["snapshot.storage.k8s.io"]
     resources: ["volumesnapshotcontents"]
     verbs: ["get", "list"]
+  - apiGroups: [""]
+    resources: ["secrets"]
+    verbs: ["get"]
 
 ---
 kind: ClusterRoleBinding
@@ -85,7 +91,7 @@ spec:
             capabilities:
               add: ["SYS_ADMIN"]
             allowPrivilegeEscalation: true
-          image: quay.io/k8scsi/csi-provisioner:v1.6.0
+          image: k8s.gcr.io/sig-storage/csi-provisioner:v2.2.2
           args:
             - --timeout=60s
             - --csi-address=$(ADDRESS)
@@ -103,7 +109,7 @@ spec:
             capabilities:
               add: ["SYS_ADMIN"]
             allowPrivilegeEscalation: true
-          image: quay.io/k8scsi/csi-attacher:v2.1.0
+          image: k8s.gcr.io/sig-storage/csi-attacher:v3.3.0
           args:
             - --v=5
             - --csi-address=$(ADDRESS)
@@ -120,7 +126,7 @@ spec:
             capabilities:
               add: ["SYS_ADMIN"]
             allowPrivilegeEscalation: true
-          image: quay.io/k8scsi/csi-resizer:v0.5.0
+          image: k8s.gcr.io/sig-storage/csi-resizer:v1.3.0
           args:
             - --v=5
             - --csi-address=$(ADDRESS)
@@ -137,7 +143,7 @@ spec:
             capabilities:
               add: ["SYS_ADMIN"]
             allowPrivilegeEscalation: true
-          image: synology/synology-csi:v1.0.0
+          image: synology/synology-csi:v1.1.0
           args:
             - --nodeid=NotUsed
             - --endpoint=$(CSI_ENDPOINT)

deploy/kubernetes/v1.19/node.yml

@@ -63,7 +63,7 @@ spec:
           securityContext:
             privileged: true
           imagePullPolicy: Always
-          image: quay.io/k8scsi/csi-node-driver-registrar:v1.2.0
+          image: k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.3.0
           args:
             - --v=5
             - --csi-address=$(ADDRESS)                         # the csi socket path inside the pod
@@ -86,7 +86,7 @@ spec:
           securityContext:
             privileged: true
           imagePullPolicy: IfNotPresent
-          image: synology/synology-csi:v1.0.0
+          image: synology/synology-csi:v1.1.0
           args:
             - --nodeid=$(KUBE_NODE_NAME)
             - --endpoint=$(CSI_ENDPOINT)

deploy/kubernetes/v1.19/snapshotter/snapshotter.yaml

@@ -64,7 +64,7 @@ spec:
             capabilities:
               add: ["SYS_ADMIN"]
             allowPrivilegeEscalation: true
-          image: quay.io/k8scsi/csi-snapshotter:v3.0.3
+          image: k8s.gcr.io/sig-storage/csi-snapshotter:v3.0.3
           args:
             - --v=5
             - --csi-address=$(ADDRESS)
@@ -81,7 +81,7 @@ spec:
             capabilities:
               add: ["SYS_ADMIN"]
             allowPrivilegeEscalation: true
-          image: synology/synology-csi:v1.0.0
+          image: synology/synology-csi:v1.1.0
           args:
             - --nodeid=NotUsed
             - --endpoint=$(CSI_ENDPOINT)

deploy/kubernetes/v1.19/snapshotter/volume-snapshot-class.yml

@@ -7,5 +7,5 @@ metadata:
 driver: csi.san.synology.com
 deletionPolicy: Delete
 # parameters:
-#   description: 'Kubernetes CSI'
+#   description: 'Kubernetes CSI' ＃ only for iscsi protocol
 #   is_locked: 'false'

deploy/kubernetes/v1.20/controller.yml

@@ -0,0 +1,168 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: csi-controller-sa
+  namespace: synology-csi
+
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: synology-csi-controller-role
+rules:
+  - apiGroups: [""]
+    resources: ["events"]
+    verbs: ["get", "list", "watch", "create", "update", "patch"]
+  - apiGroups: [""]
+    resources: ["persistentvolumeclaims"]
+    verbs: ["get", "list", "watch", "update", "patch"]
+  - apiGroups: [""]
+    resources: ["persistentvolumeclaims/status"]
+    verbs: ["get", "list", "watch", "update", "patch"]
+  - apiGroups: [""]
+    resources: ["persistentvolumes"]
+    verbs: ["get", "list", "watch", "create", "update", "patch", "delete"]
+  - apiGroups: [""]
+    resources: ["nodes"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: [""]
+    resources: ["pods"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: ["storage.k8s.io"]
+    resources: ["csinodes"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: ["csi.storage.k8s.io"]
+    resources: ["csinodeinfos"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: ["storage.k8s.io"]
+    resources: ["volumeattachments", "volumeattachments/status"]
+    verbs: ["get", "list", "watch", "update", "patch"]
+  - apiGroups: ["storage.k8s.io"]
+    resources: ["storageclasses"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshots"]
+    verbs: ["get", "list"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshotcontents"]
+    verbs: ["get", "list"]
+  - apiGroups: [""]
+    resources: ["secrets"]
+    verbs: ["get"]
+
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: synology-csi-controller-role
+  namespace: synology-csi
+subjects:
+  - kind: ServiceAccount
+    name: csi-controller-sa
+    namespace: synology-csi
+roleRef:
+  kind: ClusterRole
+  name: synology-csi-controller-role
+  apiGroup: rbac.authorization.k8s.io
+
+---
+kind: StatefulSet
+apiVersion: apps/v1
+metadata:
+  name: synology-csi-controller
+  namespace: synology-csi
+spec:
+  serviceName: "synology-csi-controller"
+  replicas: 1
+  selector:
+    matchLabels:
+      app: synology-csi-controller
+  template:
+    metadata:
+      labels:
+        app: synology-csi-controller
+    spec:
+      serviceAccountName: csi-controller-sa
+      hostNetwork: true
+      containers:
+        - name: csi-provisioner
+          securityContext:
+            privileged: true
+            capabilities:
+              add: ["SYS_ADMIN"]
+            allowPrivilegeEscalation: true
+          image: k8s.gcr.io/sig-storage/csi-provisioner:v3.0.0
+          args:
+            - --timeout=60s
+            - --csi-address=$(ADDRESS)
+            - --v=5
+          env:
+            - name: ADDRESS
+              value: /var/lib/csi/sockets/pluginproxy/csi.sock
+          imagePullPolicy: Always
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /var/lib/csi/sockets/pluginproxy/
+        - name: csi-attacher
+          securityContext:
+            privileged: true
+            capabilities:
+              add: ["SYS_ADMIN"]
+            allowPrivilegeEscalation: true
+          image: k8s.gcr.io/sig-storage/csi-attacher:v3.3.0
+          args:
+            - --v=5
+            - --csi-address=$(ADDRESS)
+          env:
+            - name: ADDRESS
+              value: /var/lib/csi/sockets/pluginproxy/csi.sock
+          imagePullPolicy: Always
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /var/lib/csi/sockets/pluginproxy/
+        - name: csi-resizer
+          securityContext:
+            privileged: true
+            capabilities:
+              add: ["SYS_ADMIN"]
+            allowPrivilegeEscalation: true
+          image: k8s.gcr.io/sig-storage/csi-resizer:v1.3.0
+          args:
+            - --v=5
+            - --csi-address=$(ADDRESS)
+          env:
+            - name: ADDRESS
+              value: /var/lib/csi/sockets/pluginproxy/csi.sock
+          imagePullPolicy: Always
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /var/lib/csi/sockets/pluginproxy/
+        - name: csi-plugin
+          securityContext:
+            privileged: true
+            capabilities:
+              add: ["SYS_ADMIN"]
+            allowPrivilegeEscalation: true
+          image: synology/synology-csi:v1.1.0
+          args:
+            - --nodeid=NotUsed
+            - --endpoint=$(CSI_ENDPOINT)
+            - --client-info
+            - /etc/synology/client-info.yml
+            - --log-level=info
+          env:
+            - name: CSI_ENDPOINT
+              value: unix:///var/lib/csi/sockets/pluginproxy/csi.sock
+          imagePullPolicy: IfNotPresent
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /var/lib/csi/sockets/pluginproxy/
+            - name: client-info
+              mountPath: /etc/synology
+              readOnly: true
+      volumes:
+        - name: socket-dir
+          emptyDir: {}
+        - name: client-info
+          secret:
+            secretName: client-info-secret

deploy/kubernetes/v1.20/csi-driver.yml

@@ -0,0 +1,9 @@
+apiVersion: storage.k8s.io/v1
+kind: CSIDriver
+metadata:
+  name: csi.san.synology.com
+spec:
+  attachRequired: true # Indicates the driver requires an attach operation (TODO: ControllerPublishVolume should be implemented)
+  podInfoOnMount: true
+  volumeLifecycleModes:
+    - Persistent

deploy/kubernetes/v1.20/namespace.yml

@@ -0,0 +1,4 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: synology-csi

deploy/kubernetes/v1.20/node.yml

@@ -0,0 +1,139 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: csi-node-sa
+  namespace: synology-csi
+
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: synology-csi-node-role
+rules:
+  - apiGroups: [""]
+    resources: ["secrets"]
+    verbs: ["get", "list"]
+  - apiGroups: [""]
+    resources: ["nodes"]
+    verbs: ["get", "list", "update"]
+  - apiGroups: [""]
+    resources: ["namespaces"]
+    verbs: ["get", "list"]
+  - apiGroups: [""]
+    resources: ["persistentvolumes"]
+    verbs: ["get", "list", "watch", "update"]
+  - apiGroups: ["storage.k8s.io"]
+    resources: ["volumeattachments"]
+    verbs: ["get", "list", "watch", "update"]
+
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: synology-csi-node-role
+  namespace: synology-csi
+subjects:
+  - kind: ServiceAccount
+    name: csi-node-sa
+    namespace: synology-csi
+roleRef:
+  kind: ClusterRole
+  name: synology-csi-node-role
+  apiGroup: rbac.authorization.k8s.io
+
+---
+kind: DaemonSet
+apiVersion: apps/v1
+metadata:
+  name: synology-csi-node
+  namespace: synology-csi
+spec:
+  selector:
+    matchLabels:
+      app: synology-csi-node
+  template:
+    metadata:
+      labels:
+        app: synology-csi-node
+    spec:
+      serviceAccount: csi-node-sa
+      hostNetwork: true
+      containers:
+        - name: csi-driver-registrar
+          securityContext:
+            privileged: true
+          imagePullPolicy: Always
+          image: k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.3.0
+          args:
+            - --v=5
+            - --csi-address=$(ADDRESS)                         # the csi socket path inside the pod
+            - --kubelet-registration-path=$(REGISTRATION_PATH) # the csi socket path on the host node
+          env:
+            - name: ADDRESS
+              value: /csi/csi.sock
+            - name: REGISTRATION_PATH
+              value: /var/lib/kubelet/plugins/csi.san.synology.com/csi.sock
+            - name: KUBE_NODE_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+          volumeMounts:
+            - name: plugin-dir
+              mountPath: /csi
+            - name: registration-dir
+              mountPath: /registration
+        - name: csi-plugin
+          securityContext:
+            privileged: true
+          imagePullPolicy: IfNotPresent
+          image: synology/synology-csi:v1.1.0
+          args:
+            - --nodeid=$(KUBE_NODE_NAME)
+            - --endpoint=$(CSI_ENDPOINT)
+            - --client-info
+            - /etc/synology/client-info.yml
+            - --log-level=info
+          env:
+            - name: CSI_ENDPOINT
+              value: unix://csi/csi.sock
+            - name: KUBE_NODE_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+          volumeMounts:
+            - name: kubelet-dir
+              mountPath: /var/lib/kubelet
+              mountPropagation: "Bidirectional"
+            - name: plugin-dir
+              mountPath: /csi
+            - name: client-info
+              mountPath: /etc/synology
+              readOnly: true
+            - name: host-root
+              mountPath: /host
+            - name: device-dir
+              mountPath: /dev
+      volumes:
+        - name: kubelet-dir
+          hostPath:
+            path: /var/lib/kubelet
+            type: Directory
+        - name: plugin-dir
+          hostPath:
+            path: /var/lib/kubelet/plugins/csi.san.synology.com/
+            type: DirectoryOrCreate
+        - name: registration-dir
+          hostPath:
+            path: /var/lib/kubelet/plugins_registry
+            type: Directory
+        - name: client-info
+          secret:
+            secretName: client-info-secret
+        - name: host-root
+          hostPath:
+            path: /
+            type: Directory
+        - name: device-dir
+          hostPath:
+            path: /dev
+            type: Directory

deploy/kubernetes/v1.20/snapshotter/snapshotter.yaml

@@ -0,0 +1,106 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: csi-snapshotter-sa
+  namespace: synology-csi
+
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: synology-csi-snapshotter-role
+rules:
+  - apiGroups: [""]
+    resources: ["events"]
+    verbs: ["list", "watch", "create", "update", "patch"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshotclasses"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshotcontents"]
+    verbs: ["create", "get", "list", "watch", "update", "delete"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshotcontents/status"]
+    verbs: ["update"]
+
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: synology-csi-snapshotter-role
+  namespace: synology-csi
+subjects:
+  - kind: ServiceAccount
+    name: csi-snapshotter-sa
+    namespace: synology-csi
+roleRef:
+  kind: ClusterRole
+  name: synology-csi-snapshotter-role
+  apiGroup: rbac.authorization.k8s.io
+
+---
+kind: StatefulSet
+apiVersion: apps/v1
+metadata:
+  name: synology-csi-snapshotter
+  namespace: synology-csi
+spec:
+  serviceName: "synology-csi-snapshotter"
+  replicas: 1
+  selector:
+    matchLabels:
+      app: synology-csi-snapshotter
+  template:
+    metadata:
+      labels:
+        app: synology-csi-snapshotter
+    spec:
+      serviceAccountName: csi-snapshotter-sa
+      hostNetwork: true
+      containers:
+        - name: csi-snapshotter
+          securityContext:
+            privileged: true
+            capabilities:
+              add: ["SYS_ADMIN"]
+            allowPrivilegeEscalation: true
+          image: k8s.gcr.io/sig-storage/csi-snapshotter:v4.2.1
+          args:
+            - --v=5
+            - --csi-address=$(ADDRESS)
+          env:
+            - name: ADDRESS
+              value: /var/lib/csi/sockets/pluginproxy/csi.sock
+          imagePullPolicy: Always
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /var/lib/csi/sockets/pluginproxy/
+        - name: csi-plugin
+          securityContext:
+            privileged: true
+            capabilities:
+              add: ["SYS_ADMIN"]
+            allowPrivilegeEscalation: true
+          image: synology/synology-csi:v1.1.0
+          args:
+            - --nodeid=NotUsed
+            - --endpoint=$(CSI_ENDPOINT)
+            - --client-info
+            - /etc/synology/client-info.yml
+            - --log-level=info
+          env:
+            - name: CSI_ENDPOINT
+              value: unix:///var/lib/csi/sockets/pluginproxy/csi.sock
+          imagePullPolicy: IfNotPresent
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /var/lib/csi/sockets/pluginproxy/
+            - name: client-info
+              mountPath: /etc/synology
+              readOnly: true
+      volumes:
+        - name: socket-dir
+          emptyDir: {}
+        - name: client-info
+          secret:
+            secretName: client-info-secret

deploy/kubernetes/v1.20/snapshotter/volume-snapshot-class.yml

@@ -0,0 +1,11 @@
+apiVersion: snapshot.storage.k8s.io/v1
+kind: VolumeSnapshotClass
+metadata:
+  name: synology-snapshotclass
+  annotations:
+    storageclass.kubernetes.io/is-default-class: "false"
+driver: csi.san.synology.com
+deletionPolicy: Delete
+# parameters:
+#   description: 'Kubernetes CSI' ＃ only for iscsi protocol
+#   is_locked: 'false'

deploy/kubernetes/v1.20/storage-class.yml

@@ -0,0 +1,14 @@
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: synology-iscsi-storage
+  # annotations:
+  #   storageclass.kubernetes.io/is-default-class: "true"
+provisioner: csi.san.synology.com
+# if all params are empty, synology CSI will choose an available location to create volume
+# parameters:
+#   dsm: '1.1.1.1'
+#   location: '/volume1'
+#   fsType: 'ext4'
+reclaimPolicy: Retain
+allowVolumeExpansion: true

deploy/kubernetes/v1.22/controller.yml

@@ -0,0 +1,279 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: csi-controller-sa
+  namespace: synology-csi
+
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: synology-csi-controller-role
+rules:
+  - apiGroups: [""]
+    resources: ["events"]
+    verbs: ["get", "list", "watch", "create", "update", "patch"]
+  - apiGroups: [""]
+    resources: ["persistentvolumeclaims"]
+    verbs: ["get", "list", "watch", "update", "patch"]
+  - apiGroups: [""]
+    resources: ["persistentvolumeclaims/status"]
+    verbs: ["get", "list", "watch", "update", "patch"]
+  - apiGroups: [""]
+    resources: ["persistentvolumes"]
+    verbs: ["get", "list", "watch", "create", "update", "patch", "delete"]
+  - apiGroups: [""]
+    resources: ["nodes"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: ["storage.k8s.io"]
+    resources: ["csinodes"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: ["csi.storage.k8s.io"]
+    resources: ["csinodeinfos"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: ["storage.k8s.io"]
+    resources: ["volumeattachments"]
+    verbs: ["get", "list", "watch", "update", "patch"]
+  - apiGroups: ["storage.k8s.io"]
+    resources: ["volumeattachments/status"]
+    verbs: ["patch"]
+  - apiGroups: ["storage.k8s.io"]
+    resources: ["storageclasses"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshots"]
+    verbs: ["get", "list"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshotcontents"]
+    verbs: ["get", "list"]
+  - apiGroups: [""]
+    resources: ["pods"]
+    verbs: ["get", "list", "watch"]
+  # The following rule should be uncommented for plugins that require secrets
+  # for provisioning.
+  - apiGroups: [""]
+    resources: ["secrets"]
+    verbs: ["get", "list", "watch"]
+
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: synology-csi-controller-role
+  namespace: synology-csi
+subjects:
+  - kind: ServiceAccount
+    name: csi-controller-sa
+    namespace: synology-csi
+roleRef:
+  kind: ClusterRole
+  name: synology-csi-controller-role
+  apiGroup: rbac.authorization.k8s.io
+
+---
+# Provisioner must be able to work with endpoints in current namespace
+# if (and only if) leadership election is enabled
+kind: Role
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  namespace: synology-csi
+  name: synology-provisioner-cfg
+rules:
+# Only one of the following rules for endpoints or leases is required based on
+# what is set for `--leader-election-type`. Endpoints are deprecated in favor of Leases.
+- apiGroups: [""]
+  resources: ["endpoints"]
+  verbs: ["get", "watch", "list", "delete", "update", "create"]
+- apiGroups: ["coordination.k8s.io"]
+  resources: ["leases"]
+  verbs: ["get", "watch", "list", "delete", "update", "create"]
+# Permissions for CSIStorageCapacity are only needed enabling the publishing
+# of storage capacity information.
+- apiGroups: ["storage.k8s.io"]
+  resources: ["csistoragecapacities"]
+  verbs: ["get", "list", "watch", "create", "update", "patch", "delete"]
+# The GET permissions below are needed for walking up the ownership chain
+# for CSIStorageCapacity. They are sufficient for deployment via
+# StatefulSet (only needs to get Pod) and Deployment (needs to get
+# Pod and then ReplicaSet to find the Deployment).
+- apiGroups: [""]
+  resources: ["pods"]
+  verbs: ["get"]
+- apiGroups: ["apps"]
+  resources: ["replicasets"]
+  verbs: ["get"]
+- apiGroups: ["apps"]
+  resources: ["statefulsets"]
+  verbs: ["get"]
+
+---
+kind: RoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: synology-csi-provisioner-role-cfg
+  namespace: synology-csi
+subjects:
+  - kind: ServiceAccount
+    name: csi-controller-sa
+    namespace: synology-csi
+roleRef:
+  kind: Role
+  name: synology-provisioner-cfg
+  apiGroup: rbac.authorization.k8s.io
+
+---
+kind: StatefulSet
+apiVersion: apps/v1
+metadata:
+  name: synology-csi-controller
+  namespace: synology-csi
+spec:
+  serviceName: "synology-csi-controller"
+  replicas: 1
+  selector:
+    matchLabels:
+      app: synology-csi-controller
+  template:
+    metadata:
+      labels:
+        app: synology-csi-controller
+    spec:
+      serviceAccountName: csi-controller-sa
+      hostNetwork: true
+      containers:
+        - name: csi-provisioner
+          securityContext:
+            privileged: true
+            capabilities:
+              add: ["SYS_ADMIN"]
+            allowPrivilegeEscalation: true
+          image: gcr.io/k8s-staging-sig-storage/csi-provisioner:v3.0.0
+          args:
+            - --timeout=60s
+            - --csi-address=$(ADDRESS)
+            - --leader-election
+#            - --enable-capacity
+            - --http-endpoint=:8080
+            - --v=1
+          env:
+            - name: ADDRESS
+              value: /var/lib/csi/sockets/pluginproxy/csi.sock
+            - name: POD_NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+            - name: POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+          imagePullPolicy: Always
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /var/lib/csi/sockets/pluginproxy/
+          ports:
+            - containerPort: 8080
+              name: prov-port
+              protocol: TCP
+          livenessProbe:
+            failureThreshold: 1
+            httpGet:
+              path: /healthz/leader-election
+              port: prov-port
+            initialDelaySeconds: 10
+            timeoutSeconds: 10
+            periodSeconds: 20
+        - name: csi-attacher
+          securityContext:
+            privileged: true
+            capabilities:
+              add: ["SYS_ADMIN"]
+            allowPrivilegeEscalation: true
+          image: gcr.io/k8s-staging-sig-storage/csi-attacher:v3.4.0
+          args:
+            - --v=1
+            - --csi-address=$(ADDRESS)
+            - --leader-election
+            - --http-endpoint=:8081
+          env:
+            - name: ADDRESS
+              value: /var/lib/csi/sockets/pluginproxy/csi.sock
+            - name: MY_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+          imagePullPolicy: Always
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /var/lib/csi/sockets/pluginproxy/
+          ports:
+            - containerPort: 8081
+              name: attach-port
+              protocol: TCP
+          livenessProbe:
+            failureThreshold: 1
+            httpGet:
+              path: /healthz/leader-election
+              port: attach-port
+            initialDelaySeconds: 10
+            timeoutSeconds: 10
+            periodSeconds: 20
+        - name: csi-resizer
+          securityContext:
+            privileged: true
+            capabilities:
+              add: ["SYS_ADMIN"]
+            allowPrivilegeEscalation: true
+          image: gcr.io/k8s-staging-sig-storage/csi-resizer:v1.3.0
+          args:
+            - --v=1
+            - --csi-address=$(ADDRESS)
+            - --leader-election
+            - --http-endpoint=:8082
+          env:
+            - name: ADDRESS
+              value: /var/lib/csi/sockets/pluginproxy/csi.sock
+          imagePullPolicy: Always
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /var/lib/csi/sockets/pluginproxy/
+          ports:
+            - containerPort: 8082
+              name: resizer-port
+              protocol: TCP
+          livenessProbe:
+            failureThreshold: 1
+            httpGet:
+              path: /healthz/leader-election
+              port: resizer-port
+            initialDelaySeconds: 10
+            timeoutSeconds: 10
+            periodSeconds: 20
+        - name: csi-plugin
+          securityContext:
+            privileged: true
+            capabilities:
+              add: ["SYS_ADMIN"]
+            allowPrivilegeEscalation: true
+          image: cristicalin/synology-csi:v1.0.0
+          args:
+            - --nodeid=NotUsed
+            - --endpoint=$(CSI_ENDPOINT)
+            - --client-info
+            - /etc/synology/client-info.yml
+            - --log-level=info
+          env:
+            - name: CSI_ENDPOINT
+              value: unix:///var/lib/csi/sockets/pluginproxy/csi.sock
+          imagePullPolicy: IfNotPresent
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /var/lib/csi/sockets/pluginproxy/
+            - name: client-info
+              mountPath: /etc/synology
+              readOnly: true
+      volumes:
+        - name: socket-dir
+          emptyDir: {}
+        - name: client-info
+          secret:
+            secretName: client-info-secret

deploy/kubernetes/v1.22/csi-driver.yml

@@ -0,0 +1,9 @@
+apiVersion: storage.k8s.io/v1
+kind: CSIDriver
+metadata:
+  name: csi.san.synology.com
+spec:
+  attachRequired: true # Indicates the driver requires an attach operation (TODO: ControllerPublishVolume should be implemented)
+  podInfoOnMount: true
+  volumeLifecycleModes:
+    - Persistent

deploy/kubernetes/v1.22/namespace.yml

@@ -0,0 +1,4 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: synology-csi

deploy/kubernetes/v1.22/node.yml

@@ -0,0 +1,139 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: csi-node-sa
+  namespace: synology-csi
+
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: synology-csi-node-role
+rules:
+  - apiGroups: [""]
+    resources: ["secrets"]
+    verbs: ["get", "list"]
+  - apiGroups: [""]
+    resources: ["nodes"]
+    verbs: ["get", "list", "update"]
+  - apiGroups: [""]
+    resources: ["namespaces"]
+    verbs: ["get", "list"]
+  - apiGroups: [""]
+    resources: ["persistentvolumes"]
+    verbs: ["get", "list", "watch", "update"]
+  - apiGroups: ["storage.k8s.io"]
+    resources: ["volumeattachments"]
+    verbs: ["get", "list", "watch", "update"]
+
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: synology-csi-node-role
+  namespace: synology-csi
+subjects:
+  - kind: ServiceAccount
+    name: csi-node-sa
+    namespace: synology-csi
+roleRef:
+  kind: ClusterRole
+  name: synology-csi-node-role
+  apiGroup: rbac.authorization.k8s.io
+
+---
+kind: DaemonSet
+apiVersion: apps/v1
+metadata:
+  name: synology-csi-node
+  namespace: synology-csi
+spec:
+  selector:
+    matchLabels:
+      app: synology-csi-node
+  template:
+    metadata:
+      labels:
+        app: synology-csi-node
+    spec:
+      serviceAccount: csi-node-sa
+      hostNetwork: true
+      containers:
+        - name: csi-driver-registrar
+          securityContext:
+            privileged: true
+          imagePullPolicy: Always
+          image: quay.io/k8scsi/csi-node-driver-registrar:v1.2.0
+          args:
+            - --v=5
+            - --csi-address=$(ADDRESS)                         # the csi socket path inside the pod
+            - --kubelet-registration-path=$(REGISTRATION_PATH) # the csi socket path on the host node
+          env:
+            - name: ADDRESS
+              value: /csi/csi.sock
+            - name: REGISTRATION_PATH
+              value: /var/lib/kubelet/plugins/csi.san.synology.com/csi.sock
+            - name: KUBE_NODE_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+          volumeMounts:
+            - name: plugin-dir
+              mountPath: /csi
+            - name: registration-dir
+              mountPath: /registration
+        - name: csi-plugin
+          securityContext:
+            privileged: true
+          imagePullPolicy: IfNotPresent
+          image: cristicalin/synology-csi:v1.0.0
+          args:
+            - --nodeid=$(KUBE_NODE_NAME)
+            - --endpoint=$(CSI_ENDPOINT)
+            - --client-info
+            - /etc/synology/client-info.yml
+            - --log-level=info
+          env:
+            - name: CSI_ENDPOINT
+              value: unix://csi/csi.sock
+            - name: KUBE_NODE_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+          volumeMounts:
+            - name: kubelet-dir
+              mountPath: /var/lib/kubelet
+              mountPropagation: "Bidirectional"
+            - name: plugin-dir
+              mountPath: /csi
+            - name: client-info
+              mountPath: /etc/synology
+              readOnly: true
+            - name: host-root
+              mountPath: /host
+            - name: device-dir
+              mountPath: /dev
+      volumes:
+        - name: kubelet-dir
+          hostPath:
+            path: /var/lib/kubelet
+            type: Directory
+        - name: plugin-dir
+          hostPath:
+            path: /var/lib/kubelet/plugins/csi.san.synology.com/
+            type: DirectoryOrCreate
+        - name: registration-dir
+          hostPath:
+            path: /var/lib/kubelet/plugins_registry
+            type: Directory
+        - name: client-info
+          secret:
+            secretName: client-info-secret
+        - name: host-root
+          hostPath:
+            path: /
+            type: Directory
+        - name: device-dir
+          hostPath:
+            path: /dev
+            type: Directory

deploy/kubernetes/v1.22/snapshotter/snapshotter.yaml

@@ -0,0 +1,157 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: csi-snapshotter-sa
+  namespace: synology-csi
+
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: synology-csi-snapshotter-role
+rules:
+  - apiGroups: [""]
+    resources: ["events"]
+    verbs: ["list", "watch", "create", "update", "patch"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshots"]
+    verbs: ["get", "list"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshotclasses"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshotcontents"]
+    verbs: ["create", "get", "list", "watch", "update", "delete"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshotcontents/status"]
+    verbs: ["update"]
+  - apiGroups: [""]
+    resources: ["secrets"]
+    verbs: ["get", "list"]
+
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: synology-csi-snapshotter-role
+  namespace: synology-csi
+subjects:
+  - kind: ServiceAccount
+    name: csi-snapshotter-sa
+    namespace: synology-csi
+roleRef:
+  kind: ClusterRole
+  name: synology-csi-snapshotter-role
+  apiGroup: rbac.authorization.k8s.io
+
+---
+kind: Role
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  namespace: synology-csi
+  name: synology-csi-snapshotter-cfg
+rules:
+- apiGroups: ["coordination.k8s.io"]
+  resources: ["leases"]
+  verbs: ["get", "watch", "list", "delete", "update", "create"]
+
+---
+kind: RoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  namespace: synology-csi
+  name: synology-csi-snapshotter-role-cfg
+subjects:
+  - kind: ServiceAccount
+    name: csi-snapshotter-sa
+    namespace: synology-csi
+roleRef:
+  kind: Role
+  name: synology-csi-snapshotter-cfg
+  apiGroup: rbac.authorization.k8s.io
+
+---
+kind: StatefulSet
+apiVersion: apps/v1
+metadata:
+  name: synology-csi-snapshotter
+  namespace: synology-csi
+spec:
+  serviceName: "synology-csi-snapshotter"
+  replicas: 1
+  selector:
+    matchLabels:
+      app: synology-csi-snapshotter
+  template:
+    metadata:
+      labels:
+        app: synology-csi-snapshotter
+    spec:
+      serviceAccountName: csi-snapshotter-sa
+      hostNetwork: true
+      containers:
+        - name: csi-snapshotter
+          securityContext:
+            privileged: true
+            capabilities:
+              add: ["SYS_ADMIN"]
+            allowPrivilegeEscalation: true
+          image: k8s.gcr.io/sig-storage/csi-snapshotter:v4.2.1
+          args:
+            - --v=1
+            - --csi-address=$(ADDRESS)
+            - --leader-election
+            - --http-endpoint=:8083
+          env:
+            - name: ADDRESS
+              value: /var/lib/csi/sockets/pluginproxy/csi.sock
+          imagePullPolicy: Always
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /var/lib/csi/sockets/pluginproxy/
+          ports:
+            - containerPort: 8083
+              name: snap-port
+              protocol: TCP
+          livenessProbe:
+            failureThreshold: 1
+            httpGet:
+              path: /healthz/leader-election
+              port: snap-port
+            initialDelaySeconds: 10
+            timeoutSeconds: 10
+            periodSeconds: 20
+        - name: csi-plugin
+          securityContext:
+            privileged: true
+            capabilities:
+              add: ["SYS_ADMIN"]
+            allowPrivilegeEscalation: true
+          image: cristicalin/synology-csi:v1.0.0
+          args:
+            - --nodeid=$(KUBE_NODE_NAME)
+            - --endpoint=$(CSI_ENDPOINT)
+            - --client-info
+            - /etc/synology/client-info.yml
+            - --log-level=info
+          env:
+            - name: CSI_ENDPOINT
+              value: unix:///var/lib/csi/sockets/pluginproxy/csi.sock
+            - name: KUBE_NODE_NAME
+              valueFrom:
+                fieldRef:
+                  apiVersion: v1
+                  fieldPath: spec.nodeName
+          imagePullPolicy: IfNotPresent
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /var/lib/csi/sockets/pluginproxy/
+            - name: client-info
+              mountPath: /etc/synology
+              readOnly: true
+      volumes:
+        - name: socket-dir
+          emptyDir: {}
+        - name: client-info
+          secret:
+            secretName: client-info-secret

deploy/kubernetes/v1.22/snapshotter/volume-snapshot-class.yml

@@ -0,0 +1,13 @@
+apiVersion: snapshot.storage.k8s.io/v1beta1
+kind: VolumeSnapshotClass
+metadata:
+  name: synology-snapshotclass
+  annotations:
+    storageclass.kubernetes.io/is-default-class: "false"
+  labels:
+    velero.io/csi-volumesnapshot-class: "true"
+driver: csi.san.synology.com
+deletionPolicy: Retain
+# parameters:
+#   description: 'Kubernetes CSI'
+#   is_locked: 'false'

deploy/kubernetes/v1.22/storage-class.yml

@@ -0,0 +1,14 @@
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: synology-iscsi-storage
+  annotations:
+    storageclass.kubernetes.io/is-default-class: "true"
+provisioner: csi.san.synology.com
+# if all params are empty, synology CSI will choose an available location to create volume
+# parameters:
+#   dsm: '1.1.1.1'
+#   location: '/volume1'
+#   fsType: 'ext4'
+reclaimPolicy: Retain
+allowVolumeExpansion: true

deploy/kubernetes/v1.23/controller.yml

@@ -0,0 +1,281 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: csi-controller-sa
+  namespace: synology-csi
+
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: synology-csi-controller-role
+rules:
+  - apiGroups: [""]
+    resources: ["events"]
+    verbs: ["get", "list", "watch", "create", "update", "patch"]
+  - apiGroups: [""]
+    resources: ["persistentvolumeclaims"]
+    verbs: ["get", "list", "watch", "update", "patch"]
+  - apiGroups: [""]
+    resources: ["persistentvolumeclaims/status"]
+    verbs: ["get", "list", "watch", "update", "patch"]
+  - apiGroups: [""]
+    resources: ["persistentvolumes"]
+    verbs: ["get", "list", "watch", "create", "update", "patch", "delete"]
+  - apiGroups: [""]
+    resources: ["nodes"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: [""]
+    resources: ["pods"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: ["storage.k8s.io"]
+    resources: ["csinodes"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: ["csi.storage.k8s.io"]
+    resources: ["csinodeinfos"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: ["storage.k8s.io"]
+    resources: ["volumeattachments", "volumeattachments/status"]
+    verbs: ["get", "list", "watch", "update", "patch"]
+  - apiGroups: ["storage.k8s.io"]
+    resources: ["storageclasses"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshots"]
+    verbs: ["get", "list"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshotcontents"]
+    verbs: ["get", "list"]
+  # The following rule should be uncommented for plugins that require secrets
+  # for provisioning.
+  - apiGroups: [""]
+    resources: ["secrets"]
+    verbs: ["get", "list", "watch"]
+
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: synology-csi-controller-role
+  namespace: synology-csi
+subjects:
+  - kind: ServiceAccount
+    name: csi-controller-sa
+    namespace: synology-csi
+roleRef:
+  kind: ClusterRole
+  name: synology-csi-controller-role
+  apiGroup: rbac.authorization.k8s.io
+
+---
+# Provisioner must be able to work with endpoints in current namespace
+# if (and only if) leadership election is enabled
+kind: Role
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  namespace: synology-csi
+  name: synology-provisioner-cfg
+rules:
+# Only one of the following rules for endpoints or leases is required based on
+# what is set for `--leader-election-type`. Endpoints are deprecated in favor of Leases.
+- apiGroups: [""]
+  resources: ["endpoints"]
+  verbs: ["get", "watch", "list", "delete", "update", "create"]
+- apiGroups: ["coordination.k8s.io"]
+  resources: ["leases"]
+  verbs: ["get", "watch", "list", "delete", "update", "create"]
+# Permissions for CSIStorageCapacity are only needed enabling the publishing
+# of storage capacity information.
+- apiGroups: ["storage.k8s.io"]
+  resources: ["csistoragecapacities"]
+  verbs: ["get", "list", "watch", "create", "update", "patch", "delete"]
+# The GET permissions below are needed for walking up the ownership chain
+# for CSIStorageCapacity. They are sufficient for deployment via
+# StatefulSet (only needs to get Pod) and Deployment (needs to get
+# Pod and then ReplicaSet to find the Deployment).
+- apiGroups: [""]
+  resources: ["pods"]
+  verbs: ["get"]
+- apiGroups: ["apps"]
+  resources: ["replicasets"]
+  verbs: ["get"]
+- apiGroups: ["apps"]
+  resources: ["statefulsets"]
+  verbs: ["get"]
+
+---
+kind: RoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: synology-csi-provisioner-role-cfg
+  namespace: synology-csi
+subjects:
+  - kind: ServiceAccount
+    name: csi-controller-sa
+    namespace: synology-csi
+roleRef:
+  kind: Role
+  name: synology-provisioner-cfg
+  apiGroup: rbac.authorization.k8s.io
+
+---
+kind: StatefulSet
+apiVersion: apps/v1
+metadata:
+  name: synology-csi-controller
+  namespace: synology-csi
+spec:
+  serviceName: "synology-csi-controller"
+  replicas: 1
+  selector:
+    matchLabels:
+      app: synology-csi-controller
+  template:
+    metadata:
+      labels:
+        app: synology-csi-controller
+    spec:
+      serviceAccountName: csi-controller-sa
+      hostNetwork: true
+      containers:
+        - name: csi-provisioner
+          securityContext:
+            privileged: true
+            capabilities:
+              add: ["SYS_ADMIN"]
+            allowPrivilegeEscalation: true
+          image: k8s.gcr.io/sig-storage/csi-provisioner:v3.1.0
+          args:
+            - --timeout=60s
+            - --csi-address=$(ADDRESS)
+            - --leader-election
+#            - --enable-capacity
+            - --http-endpoint=:8080
+            - --v=5
+          env:
+            - name: ADDRESS
+              value: /var/lib/csi/sockets/pluginproxy/csi.sock
+            - name: POD_NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+            - name: POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+          imagePullPolicy: Always
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /var/lib/csi/sockets/pluginproxy/
+          ports:
+            - containerPort: 8080
+              name: prov-port
+              protocol: TCP
+          livenessProbe:
+            failureThreshold: 1
+            httpGet:
+              path: /healthz/leader-election
+              port: prov-port
+            initialDelaySeconds: 10
+            timeoutSeconds: 10
+            periodSeconds: 20
+        - name: csi-attacher
+          securityContext:
+            privileged: true
+            capabilities:
+              add: ["SYS_ADMIN"]
+            allowPrivilegeEscalation: true
+          image: k8s.gcr.io/sig-storage/csi-attacher:v3.4.0
+          args:
+            - --v=5
+            - --csi-address=$(ADDRESS)
+            - --leader-election
+            - --http-endpoint=:8081
+          env:
+            - name: ADDRESS
+              value: /var/lib/csi/sockets/pluginproxy/csi.sock
+            - name: MY_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+          imagePullPolicy: Always
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /var/lib/csi/sockets/pluginproxy/
+          ports:
+            - containerPort: 8081
+              name: attach-port
+              protocol: TCP
+          livenessProbe:
+            failureThreshold: 1
+            httpGet:
+              path: /healthz/leader-election
+              port: attach-port
+            initialDelaySeconds: 10
+            timeoutSeconds: 10
+            periodSeconds: 20
+        - name: csi-resizer
+          securityContext:
+            privileged: true
+            capabilities:
+              add: ["SYS_ADMIN"]
+            allowPrivilegeEscalation: true
+          image: k8s.gcr.io/sig-storage/csi-resizer:v1.4.0
+          args:
+            - --v=5
+            - --csi-address=$(ADDRESS)
+            - --leader-election
+            - --http-endpoint=:8082
+          env:
+            - name: ADDRESS
+              value: /var/lib/csi/sockets/pluginproxy/csi.sock
+          imagePullPolicy: Always
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /var/lib/csi/sockets/pluginproxy/
+          ports:
+            - containerPort: 8082
+              name: resizer-port
+              protocol: TCP
+          livenessProbe:
+            failureThreshold: 1
+            httpGet:
+              path: /healthz/leader-election
+              port: resizer-port
+            initialDelaySeconds: 10
+            timeoutSeconds: 10
+            periodSeconds: 20
+        - name: csi-plugin
+          securityContext:
+            privileged: true
+            capabilities:
+              add: ["SYS_ADMIN"]
+            allowPrivilegeEscalation: true
+          image: cristicalin/synology-csi:v1.1.0
+          args:
+            - --nodeid=$(KUBE_NODE_NAME)
+            - --endpoint=$(CSI_ENDPOINT)
+            - --client-info
+            - /etc/synology/client-info.yml
+            - --log-level=info
+          env:
+            - name: CSI_ENDPOINT
+              value: unix:///var/lib/csi/sockets/pluginproxy/csi.sock
+            - name: KUBE_NODE_NAME
+              valueFrom:
+                fieldRef:
+                  apiVersion: v1
+                  fieldPath: spec.nodeName
+          imagePullPolicy: IfNotPresent
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /var/lib/csi/sockets/pluginproxy/
+            - name: client-info
+              mountPath: /etc/synology
+              readOnly: true
+      volumes:
+        - name: socket-dir
+          emptyDir: {}
+        - name: client-info
+          secret:
+            secretName: client-info-secret

deploy/kubernetes/v1.23/csi-driver.yml

@@ -0,0 +1,11 @@
+apiVersion: storage.k8s.io/v1
+kind: CSIDriver
+metadata:
+  name: csi.san.synology.com
+spec:
+  attachRequired: true # Indicates the driver requires an attach operation (TODO: ControllerPublishVolume should be implemented)
+  podInfoOnMount: true
+#  fsGroupPolicy: File (see https://kubernetes-csi.github.io/docs/support-fsgroup.html#supported-modes)
+  fsGroupPolicy: ReadWriteOnceWithFSType
+  volumeLifecycleModes:
+    - Persistent

deploy/kubernetes/v1.23/namespace.yml

@@ -0,0 +1,4 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: synology-csi

deploy/kubernetes/v1.23/node.yml

@@ -0,0 +1,150 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: csi-node-sa
+  namespace: synology-csi
+
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: synology-csi-node-role
+rules:
+  - apiGroups: [""]
+    resources: ["secrets"]
+    verbs: ["get", "list"]
+  - apiGroups: [""]
+    resources: ["nodes"]
+    verbs: ["get", "list", "update"]
+  - apiGroups: [""]
+    resources: ["namespaces"]
+    verbs: ["get", "list"]
+  - apiGroups: [""]
+    resources: ["persistentvolumes"]
+    verbs: ["get", "list", "watch", "update"]
+  - apiGroups: ["storage.k8s.io"]
+    resources: ["volumeattachments"]
+    verbs: ["get", "list", "watch", "update"]
+
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: synology-csi-node-role
+  namespace: synology-csi
+subjects:
+  - kind: ServiceAccount
+    name: csi-node-sa
+    namespace: synology-csi
+roleRef:
+  kind: ClusterRole
+  name: synology-csi-node-role
+  apiGroup: rbac.authorization.k8s.io
+
+---
+kind: DaemonSet
+apiVersion: apps/v1
+metadata:
+  name: synology-csi-node
+  namespace: synology-csi
+spec:
+  selector:
+    matchLabels:
+      app: synology-csi-node
+  template:
+    metadata:
+      labels:
+        app: synology-csi-node
+    spec:
+      serviceAccount: csi-node-sa
+      hostNetwork: true
+      containers:
+        - name: csi-driver-registrar
+          securityContext:
+            privileged: true
+          imagePullPolicy: Always
+          image: k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.5.1
+#          image: k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.5.0
+          args:
+            - --v=5
+            - --csi-address=$(ADDRESS)                         # the csi socket path inside the pod
+            - --kubelet-registration-path=$(REGISTRATION_PATH) # the csi socket path on the host node
+            - --health-port=9809
+          env:
+            - name: ADDRESS
+              value: /csi/csi.sock
+            - name: REGISTRATION_PATH
+              value: /var/lib/kubelet/plugins/csi.san.synology.com/csi.sock
+            - name: KUBE_NODE_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+          volumeMounts:
+            - name: plugin-dir
+              mountPath: /csi
+            - name: registration-dir
+              mountPath: /registration
+          ports:
+            - containerPort: 9809
+              name: healthz
+          livenessProbe:
+            httpGet:
+              path: /healthz
+              port: healthz
+            initialDelaySeconds: 5
+            timeoutSeconds: 5
+        - name: csi-plugin
+          securityContext:
+            privileged: true
+          imagePullPolicy: IfNotPresent
+          image: cristicalin/synology-csi:v1.1.0
+          args:
+            - --nodeid=$(KUBE_NODE_NAME)
+            - --endpoint=$(CSI_ENDPOINT)
+            - --client-info
+            - /etc/synology/client-info.yml
+            - --log-level=info
+          env:
+            - name: CSI_ENDPOINT
+              value: unix://csi/csi.sock
+            - name: KUBE_NODE_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+          volumeMounts:
+            - name: kubelet-dir
+              mountPath: /var/lib/kubelet
+              mountPropagation: "Bidirectional"
+            - name: plugin-dir
+              mountPath: /csi
+            - name: client-info
+              mountPath: /etc/synology
+              readOnly: true
+            - name: host-root
+              mountPath: /host
+            - name: device-dir
+              mountPath: /dev
+      volumes:
+        - name: kubelet-dir
+          hostPath:
+            path: /var/lib/kubelet
+            type: Directory
+        - name: plugin-dir
+          hostPath:
+            path: /var/lib/kubelet/plugins/csi.san.synology.com/
+            type: DirectoryOrCreate
+        - name: registration-dir
+          hostPath:
+            path: /var/lib/kubelet/plugins_registry
+            type: Directory
+        - name: client-info
+          secret:
+            secretName: client-info-secret
+        - name: host-root
+          hostPath:
+            path: /
+            type: Directory
+        - name: device-dir
+          hostPath:
+            path: /dev
+            type: Directory

deploy/kubernetes/v1.23/snapshotter/snapshotter.yaml

@@ -0,0 +1,157 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: csi-snapshotter-sa
+  namespace: synology-csi
+
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: synology-csi-snapshotter-role
+rules:
+  - apiGroups: [""]
+    resources: ["events"]
+    verbs: ["list", "watch", "create", "update", "patch"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshots"]
+    verbs: ["get", "list"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshotclasses"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshotcontents"]
+    verbs: ["create", "get", "list", "watch", "update", "delete", "patch"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshotcontents/status"]
+    verbs: ["update"]
+  - apiGroups: [""]
+    resources: ["secrets"]
+    verbs: ["get", "list"]
+
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: synology-csi-snapshotter-role
+  namespace: synology-csi
+subjects:
+  - kind: ServiceAccount
+    name: csi-snapshotter-sa
+    namespace: synology-csi
+roleRef:
+  kind: ClusterRole
+  name: synology-csi-snapshotter-role
+  apiGroup: rbac.authorization.k8s.io
+
+---
+kind: Role
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  namespace: synology-csi
+  name: synology-csi-snapshotter-cfg
+rules:
+- apiGroups: ["coordination.k8s.io"]
+  resources: ["leases"]
+  verbs: ["get", "watch", "list", "delete", "update", "create"]
+
+---
+kind: RoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  namespace: synology-csi
+  name: synology-csi-snapshotter-role-cfg
+subjects:
+  - kind: ServiceAccount
+    name: csi-snapshotter-sa
+    namespace: synology-csi
+roleRef:
+  kind: Role
+  name: synology-csi-snapshotter-cfg
+  apiGroup: rbac.authorization.k8s.io
+
+---
+kind: StatefulSet
+apiVersion: apps/v1
+metadata:
+  name: synology-csi-snapshotter
+  namespace: synology-csi
+spec:
+  serviceName: "synology-csi-snapshotter"
+  replicas: 1
+  selector:
+    matchLabels:
+      app: synology-csi-snapshotter
+  template:
+    metadata:
+      labels:
+        app: synology-csi-snapshotter
+    spec:
+      serviceAccountName: csi-snapshotter-sa
+      hostNetwork: true
+      containers:
+        - name: csi-snapshotter
+          securityContext:
+            privileged: true
+            capabilities:
+              add: ["SYS_ADMIN"]
+            allowPrivilegeEscalation: true
+          image: k8s.gcr.io/sig-storage/csi-snapshotter:v5.0.1
+          args:
+            - --v=5
+            - --csi-address=$(ADDRESS)
+            - --leader-election
+            - --http-endpoint=:8083
+          env:
+            - name: ADDRESS
+              value: /var/lib/csi/sockets/pluginproxy/csi.sock
+          imagePullPolicy: Always
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /var/lib/csi/sockets/pluginproxy/
+          ports:
+            - containerPort: 8083
+              name: snap-port
+              protocol: TCP
+          livenessProbe:
+            failureThreshold: 1
+            httpGet:
+              path: /healthz/leader-election
+              port: snap-port
+            initialDelaySeconds: 10
+            timeoutSeconds: 10
+            periodSeconds: 20
+        - name: csi-plugin
+          securityContext:
+            privileged: true
+            capabilities:
+              add: ["SYS_ADMIN"]
+            allowPrivilegeEscalation: true
+          image: cristicalin/synology-csi:v1.1.0
+          args:
+            - --nodeid=$(KUBE_NODE_NAME)
+            - --endpoint=$(CSI_ENDPOINT)
+            - --client-info
+            - /etc/synology/client-info.yml
+            - --log-level=info
+          env:
+            - name: CSI_ENDPOINT
+              value: unix:///var/lib/csi/sockets/pluginproxy/csi.sock
+            - name: KUBE_NODE_NAME
+              valueFrom:
+                fieldRef:
+                  apiVersion: v1
+                  fieldPath: spec.nodeName
+          imagePullPolicy: IfNotPresent
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /var/lib/csi/sockets/pluginproxy/
+            - name: client-info
+              mountPath: /etc/synology
+              readOnly: true
+      volumes:
+        - name: socket-dir
+          emptyDir: {}
+        - name: client-info
+          secret:
+            secretName: client-info-secret

deploy/kubernetes/v1.23/snapshotter/volume-snapshot-class.yml

@@ -0,0 +1,13 @@
+apiVersion: snapshot.storage.k8s.io/v1
+kind: VolumeSnapshotClass
+metadata:
+  name: synology-snapshotclass
+  annotations:
+    storageclass.kubernetes.io/is-default-class: "false"
+  labels:
+    velero.io/csi-volumesnapshot-class: "true"
+driver: csi.san.synology.com
+deletionPolicy: Retain
+# parameters:
+#   description: 'Kubernetes CSI'
+#   is_locked: 'false'

deploy/kubernetes/v1.23/storage-class.yml

@@ -0,0 +1,14 @@
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: synology-iscsi-storage
+  annotations:
+    storageclass.kubernetes.io/is-default-class: "true"
+provisioner: csi.san.synology.com
+# if all params are empty, synology CSI will choose an available location to create volume
+# parameters:
+#   dsm: '1.1.1.1'
+#   location: '/volume1'
+#   fsType: 'ext4'
+reclaimPolicy: Retain
+allowVolumeExpansion: true

go.mod

@@ -5,17 +5,23 @@ go 1.16
 require (
 	github.com/antonfisher/nested-logrus-formatter v1.3.1
 	github.com/cenkalti/backoff/v4 v4.1.1
-	github.com/container-storage-interface/spec v1.3.0
-	github.com/golang/protobuf v1.4.3
-	github.com/kubernetes-csi/csi-lib-utils v0.9.1
+	github.com/container-storage-interface/spec v1.5.0
+	github.com/go-logr/logr v1.1.0 // indirect
+	github.com/golang/protobuf v1.5.2
+	github.com/kubernetes-csi/csi-lib-utils v0.10.0
 	github.com/kubernetes-csi/csi-test/v4 v4.2.0
-	github.com/sirupsen/logrus v1.7.0
-	github.com/spf13/cobra v1.1.3
-	github.com/spf13/pflag v1.0.5
-	google.golang.org/grpc v1.34.0
+	github.com/sirupsen/logrus v1.8.1
+	github.com/spf13/cobra v1.2.1
+	golang.org/x/net v0.0.0-20210908191846-a5e095526f91 // indirect
+	golang.org/x/sys v0.0.0-20210910150752-751e447fb3d0 // indirect
+	golang.org/x/text v0.3.7 // indirect
+	google.golang.org/genproto v0.0.0-20210909211513-a8c4777a87af // indirect
+	google.golang.org/grpc v1.40.0
+	google.golang.org/protobuf v1.27.1 // indirect
 	gopkg.in/yaml.v2 v2.4.0
-	k8s.io/mount-utils v0.21.2
-	k8s.io/utils v0.0.0-20210527160623-6fdb442a123b
+	k8s.io/klog/v2 v2.20.0 // indirect
+	k8s.io/mount-utils v0.22.1
+	k8s.io/utils v0.0.0-20210820185131-d34e5cb4466e
 )
 
 exclude google.golang.org/grpc v1.37.0

go.sum

@@ -5,46 +5,73 @@ cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6A
 cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY=
 cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc=
 cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0=
-cloud.google.com/go v0.51.0/go.mod h1:hWtGJ6gnXH+KgDv+V0zFGDvpi07n3z8ZNj3T1RW0Gcw=
+cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To=
+cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4=
+cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M=
+cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc=
+cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk=
+cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs=
+cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc=
+cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY=
+cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI=
+cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk=
+cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg=
+cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8=
+cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0=
 cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
+cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE=
+cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc=
+cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg=
+cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc=
+cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ=
 cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
+cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk=
 cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk=
 cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=
+cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw=
+cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA=
+cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU=
 cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw=
+cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos=
+cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk=
+cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs=
+cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0=
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
-github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8=
-github.com/Azure/go-autorest/autorest v0.9.0/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI=
-github.com/Azure/go-autorest/autorest v0.9.6/go.mod h1:/FALq9T/kS7b5J5qsQ+RSTUdAmGFqi0vUdVNNx8q630=
-github.com/Azure/go-autorest/autorest/adal v0.5.0/go.mod h1:8Z9fGy2MpX0PvDjB1pEgQTmVqjGhiHBW7RJJEciWzS0=
-github.com/Azure/go-autorest/autorest/adal v0.8.2/go.mod h1:ZjhuQClTqx435SRJ2iMlOxPYt3d2C/T/7TiQCVZSn3Q=
-github.com/Azure/go-autorest/autorest/date v0.1.0/go.mod h1:plvfp3oPSKwf2DNjlBjWF/7vwR+cUD/ELuzDCXwHUVA=
-github.com/Azure/go-autorest/autorest/date v0.2.0/go.mod h1:vcORJHLJEh643/Ioh9+vPmf1Ij9AEBM5FuBIXLmIy0g=
-github.com/Azure/go-autorest/autorest/mocks v0.1.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0=
-github.com/Azure/go-autorest/autorest/mocks v0.2.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0=
-github.com/Azure/go-autorest/autorest/mocks v0.3.0/go.mod h1:a8FDP3DYzQ4RYfVAxAN3SVSiiO77gL2j2ronKKP0syM=
-github.com/Azure/go-autorest/logger v0.1.0/go.mod h1:oExouG+K6PryycPJfVSxi/koC6LSNgds39diKLz7Vrc=
-github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk=
+github.com/Azure/go-ansiterm v0.0.0-20210608223527-2377c96fe795/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8=
+github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E=
+github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24=
+github.com/Azure/go-autorest/autorest v0.11.18/go.mod h1:dSiJPy22c3u0OtOKDNttNgqpNFY/GeWa7GH/Pz56QRA=
+github.com/Azure/go-autorest/autorest/adal v0.9.13/go.mod h1:W/MM4U6nLxnIskrw4UwWzlHfGjwUS50aOsc/I3yuU8M=
+github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74=
+github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k=
+github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8=
+github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
 github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ=
 github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
-github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
-github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
+github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
+github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
 github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
+github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho=
+github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
 github.com/antonfisher/nested-logrus-formatter v1.3.1 h1:NFJIr+pzwv5QLHTPyKz9UMEoHck02Q9L0FP13b/xSbQ=
 github.com/antonfisher/nested-logrus-formatter v1.3.1/go.mod h1:6WTfyWFkBc9+zyBaKIqRrg/KwMqBbodBjgbHjDz7zjA=
 github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
 github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
 github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
+github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
+github.com/benbjohnson/clock v1.0.3/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
 github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
 github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84=
-github.com/blang/semver v3.5.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
+github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqOes/6LfM=
+github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
 github.com/cenkalti/backoff/v4 v4.1.1 h1:G2HAfAmvm/GcKan2oOQpBXOd2tT2G57ZnZGWa1PxPBQ=
 github.com/cenkalti/backoff/v4 v4.1.1/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
@@ -56,22 +83,25 @@ github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMn
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
 github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
 github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
-github.com/container-storage-interface/spec v1.2.0/go.mod h1:6URME8mwIBbpVyZV93Ce5St17xBiQJQY67NDsuohiy4=
-github.com/container-storage-interface/spec v1.3.0 h1:wMH4UIoWnK/TXYw8mbcIHgZmB6kHOeIsYsiaTJwa6bc=
+github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
+github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
 github.com/container-storage-interface/spec v1.3.0/go.mod h1:6URME8mwIBbpVyZV93Ce5St17xBiQJQY67NDsuohiy4=
+github.com/container-storage-interface/spec v1.5.0 h1:lvKxe3uLgqQeVQcrnL2CPQKISoKjTJxojEs9cBk+HXo=
+github.com/container-storage-interface/spec v1.5.0/go.mod h1:8K96oQNkJ7pFcC2R9Z1ynGGBB1I93kcS6PGg3SsOk8s=
 github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
 github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
 github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
 github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
+github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
 github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
 github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
+github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
 github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
-github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM=
 github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE=
 github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc=
 github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
@@ -79,45 +109,63 @@ github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymF
 github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
 github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po=
+github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
+github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
+github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
-github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
+github.com/evanphx/json-patch v4.11.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
 github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
+github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
+github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k=
+github.com/form3tech-oss/jwt-go v3.2.3+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
 github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
-github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
 github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
+github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
+github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY=
 github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
+github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A=
 github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas=
 github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU=
 github.com/go-logr/logr v0.3.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU=
-github.com/go-logr/logr v0.4.0 h1:K7/B1jt6fIBQVd4Owv2MqGQClcgf0R266+7C/QjRcLc=
 github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU=
-github.com/go-openapi/jsonpointer v0.0.0-20160704185906-46af16f9f7b1/go.mod h1:+35s3my2LFTysnkMfxsJBAMHj/DoqoB9knIWoYG/Vk0=
-github.com/go-openapi/jsonreference v0.0.0-20160704190145-13c6e3589ad9/go.mod h1:W3Z9FmVs9qj+KR4zFKmDPGiLdk1D9Rlm7cyMvf57TTg=
-github.com/go-openapi/spec v0.0.0-20160808142527-6aced65f8501/go.mod h1:J8+jY1nAiCcj+friV/PDoE1/3eeccG9LYBs0tYvLOWc=
-github.com/go-openapi/swag v0.0.0-20160704191624-1d0bd113de87/go.mod h1:DXUve3Dpr1UfpPtxFw+EFuQ41HhCWZfha5jSVRG7C7I=
+github.com/go-logr/logr v1.0.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
+github.com/go-logr/logr v1.1.0 h1:nAbevmWlS2Ic4m4+/An5NXkaGqlqpbBgdcuThZxnZyI=
+github.com/go-logr/logr v1.1.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
+github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
+github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8=
+github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
+github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
 github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
-github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
+github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
 github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
+github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
+github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
+github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
 github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4=
+github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
+github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
+github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk=
 github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
 github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
 github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
@@ -125,35 +173,57 @@ github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:W
 github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
 github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
 github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
-github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM=
 github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
+github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
+github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM=
+github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw=
+github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
 github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
+github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.0 h1:/QaMHBdZ26BB3SSst0Iwl10Epc+xhTquomWX0oZEB6w=
+github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU=
+github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
+github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
+github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
 github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
 github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
 github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
+github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
+github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
+github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
 github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
 github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y=
 github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
 github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
-github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg=
+github.com/googleapis/gnostic v0.5.1/go.mod h1:6U4PtQXGIEt/Z3h5MAT7FNofLnw9vXk2cUuW7uA/OeU=
+github.com/googleapis/gnostic v0.5.5/go.mod h1:7+EbHbldMins07ALC74bsA81Ovc97DwqyJO1AENw9kA=
 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
 github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
 github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
 github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
 github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
+github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=
 github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q=
 github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8=
 github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
@@ -176,21 +246,26 @@ github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2p
 github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
 github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
+github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
 github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
 github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
 github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
+github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=
 github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
 github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
+github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
 github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
 github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
 github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
+github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
 github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
-github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
+github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=
 github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
@@ -198,12 +273,14 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
-github.com/kubernetes-csi/csi-lib-utils v0.9.1 h1:sGq6ifVujfMSkfTsMZip44Ttv8SDXvsBlFk9GdYl/b8=
-github.com/kubernetes-csi/csi-lib-utils v0.9.1/go.mod h1:8E2jVUX9j3QgspwHXa6LwyN7IHQDjW9jX3kwoWnSC+M=
+github.com/kubernetes-csi/csi-lib-utils v0.10.0 h1:Aqm8X81eCzzfH/bvIEqSWtcbK9HF9NbFk4d+le1snVA=
+github.com/kubernetes-csi/csi-lib-utils v0.10.0/go.mod h1:BmGZZB16L18+9+Lgg9YWwBKfNEHIDdgGfAyuW6p2NV0=
 github.com/kubernetes-csi/csi-test/v4 v4.2.0 h1:uyFJMSN9vnOOuQwndB43Kp4Bi/dScuATdv4FMuGJJQ8=
 github.com/kubernetes-csi/csi-test/v4 v4.2.0/go.mod h1:HuWP7lCCJzehodzd4kO170soxqgzSQHZ5Jbp1pKPlmA=
 github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
-github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
+github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60=
+github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
+github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
 github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
 github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
@@ -217,13 +294,16 @@ github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS4
 github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY=
 github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
 github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
-github.com/moby/term v0.0.0-20200312100748-672ec06f55cd/go.mod h1:DdlQx2hp0Ss5/fLikoLlEeIYiATotOjgB//nb973jeo=
+github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
+github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c=
+github.com/moby/term v0.0.0-20210610120745-9d4ed1856297/go.mod h1:vgPCkQMyxTZ7IDy8SXRufE172gr8+K/JE/7hHFxHW3A=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
+github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw=
 github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs=
 github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
@@ -233,22 +313,23 @@ github.com/nxadm/tail v1.4.5/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI
 github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
 github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
-github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
+github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY=
 github.com/onsi/ginkgo v1.14.2 h1:8mVmC9kjFFmA8H4pKMUhcblgifdkOIXPvbhN1T36q1M=
 github.com/onsi/ginkgo v1.14.2/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY=
 github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
-github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
 github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
 github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
 github.com/onsi/gomega v1.10.4 h1:NiTx7EEvBzu9sFOD1zORteLSt3o8gnlvZZwSE9TnY9U=
 github.com/onsi/gomega v1.10.4/go.mod h1:g/HbgYopi++010VEqkFgJHKC09uJiW9UkXvMUuKHUCQ=
 github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
+github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c=
 github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI=
@@ -256,6 +337,7 @@ github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXP
 github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso=
 github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
 github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M=
+github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0=
 github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
 github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
@@ -264,13 +346,16 @@ github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7q
 github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
 github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
 github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo=
+github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc=
 github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
 github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
 github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
 github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
+github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
 github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
 github.com/robertkrimen/otto v0.0.0-20200922221731-ef014fd054ac/go.mod h1:xvqspoSXJTIpemEonrMDFq6XzwHYYgToXWj5eRX1OtY=
 github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
+github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
@@ -279,55 +364,95 @@ github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeV
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
 github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
-github.com/sirupsen/logrus v1.7.0 h1:ShrD1U9pZB12TX0cVy0DtePoCH97K8EtX+mg7ZARUtM=
 github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
+github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE=
+github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
 github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
 github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
 github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
 github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
 github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
 github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk=
+github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I=
 github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
-github.com/spf13/cobra v1.1.3 h1:xghbfqPkxzxP3C/f3n5DdpAbdKLj4ZE4BWQI362l53M=
+github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
 github.com/spf13/cobra v1.1.3/go.mod h1:pGADOWyqRD/YMrPZigI/zbliZ2wVD/23d+is3pSWzOo=
+github.com/spf13/cobra v1.2.1 h1:+KmjbUw1hriSNMF55oPrkZcb27aECyrj8V2ytv7kWDw=
+github.com/spf13/cobra v1.2.1/go.mod h1:ExllRjgxM/piMAM+3tAZvg8fsklGAf3tPfi+i8t68Nk=
 github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
+github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo=
 github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
 github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
 github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg=
+github.com/spf13/viper v1.8.1/go.mod h1:o0Pch8wJ9BVSWGQMbra6iw0oQ5oktSIBaujf1rJH9Ns=
+github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
-github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0=
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
+github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
 go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
+go.etcd.io/etcd/api/v3 v3.5.0/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs=
+go.etcd.io/etcd/client/pkg/v3 v3.5.0/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g=
+go.etcd.io/etcd/client/v2 v2.305.0/go.mod h1:h9puh54ZTgAKtEbut2oe9P4L/oqKCVB6xsXlzd7alYQ=
 go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
 go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
 go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
+go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
+go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
+go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk=
+go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E=
+go.opentelemetry.io/contrib v0.20.0/go.mod h1:G/EtFaa6qaN7+LxqfIAT3GiZa7Wv5DTBUzl5H4LY0Kc=
+go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.20.0/go.mod h1:2AboqHi0CiIZU0qwhtUfCYD1GeUzvvIXWNkhDt7ZMG4=
+go.opentelemetry.io/otel v0.20.0/go.mod h1:Y3ugLH2oa81t5QO+Lty+zXf8zC9L26ax4Nzoxm/dooo=
+go.opentelemetry.io/otel/exporters/otlp v0.20.0/go.mod h1:YIieizyaN77rtLJra0buKiNBOm9XQfkPEKBeuhoMwAM=
+go.opentelemetry.io/otel/metric v0.20.0/go.mod h1:598I5tYlH1vzBjn+BTuhzTCSb/9debfNp6R3s7Pr1eU=
+go.opentelemetry.io/otel/oteltest v0.20.0/go.mod h1:L7bgKf9ZB7qCwT9Up7i9/pn0PWIa9FqQ2IQ8LoxiGnw=
+go.opentelemetry.io/otel/sdk v0.20.0/go.mod h1:g/IcepuwNsoiX5Byy2nNV0ySUF1em498m7hBWC279Yc=
+go.opentelemetry.io/otel/sdk/export/metric v0.20.0/go.mod h1:h7RBNMsDJ5pmI1zExLi+bJK+Dr8NQCh0qGhm1KDnNlE=
+go.opentelemetry.io/otel/sdk/metric v0.20.0/go.mod h1:knxiS8Xd4E/N+ZqKmUPf3gTTZ4/0TjTXukfxjzSTpHE=
+go.opentelemetry.io/otel/trace v0.20.0/go.mod h1:6GjCW8zgDjwGHGa6GkyeB8+/5vjT16gUEi0Nf1iBdgw=
+go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI=
 go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
+go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
 go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
+go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
 go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
+go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo=
 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
 golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek=
 golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY=
+golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
 golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
+golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
+golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
+golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
 golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
 golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
@@ -338,11 +463,21 @@ golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHl
 golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
 golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
 golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs=
+golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
+golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
+golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
+golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
 golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
 golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
 golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
 golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY=
 golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
+golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
+golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -359,23 +494,59 @@ golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn
 golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
 golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
 golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
+golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
+golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
-golang.org/x/net v0.0.0-20201209123823-ac852fbbde11 h1:lwlPPsmjDKK0J6eG6xDWd5XPehI0R024zxjDnw3esPA=
 golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc=
+golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
+golang.org/x/net v0.0.0-20210520170846-37e1c6afe023/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20210908191846-a5e095526f91 h1:E8wdt+zBjoxD3MA65wEc3pl25BsTi7tbkpwc4ANThjc=
+golang.org/x/net v0.0.0-20210908191846-a5e095526f91/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
+golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
+golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
+golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
+golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
+golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
+golang.org/x/oauth2 v0.0.0-20210402161424-2e8d93401602/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -391,36 +562,71 @@ golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200622214017-ed371f2e16b4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200831180312-196b9ba8737a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20201207223542-d4d67f95c62d h1:MiWWjyhUzZ+jvhZvloX6ZrUsdEghn8a64Upd8EMHglE=
+golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201207223542-d4d67f95c62d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210910150752-751e447fb3d0 h1:xrCZDmdtoloIiooiA9q0OQb9r8HejIHYoHGhGCe1pGg=
+golang.org/x/sys v0.0.0-20210910150752-751e447fb3d0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.3.4 h1:0YWbFKbhXG/wIiuHDSKpS0Iy7FSA+u45VtBMfQcFTTc=
 golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=
+golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
-golang.org/x/tools v0.0.0-20181011042414-1f849cf54d09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
-golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
@@ -438,8 +644,41 @@ golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtn
 golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
 golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw=
+golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw=
+golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8=
+golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
+golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
+golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
+golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE=
+golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
+golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
+golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
+golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
+golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
+golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
+golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
+golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -450,12 +689,30 @@ google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E
 google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
 google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
 google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
+google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
 google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
+google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE=
+google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE=
+google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM=
+google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc=
+google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg=
+google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE=
+google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8=
+google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU=
+google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94=
+google.golang.org/api v0.44.0/go.mod h1:EBOGZqzyhtvMDoxwS97ctnh0zUmYY6CxqXsc1AvkYD8=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
 google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
+google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
+google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
 google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
 google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
 google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
@@ -465,10 +722,42 @@ google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98
 google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
 google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8=
 google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
 google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA=
+google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U=
 google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
-google.golang.org/genproto v0.0.0-20201209185603-f92720507ed4 h1:J4dpx/41slnq1aogzUSTuBuvD7VXz7ZLkVpr32YgSlg=
+google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA=
+google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20201019141844-1ed22bb0c154/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
 google.golang.org/genproto v0.0.0-20201209185603-f92720507ed4/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A=
+google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0=
+google.golang.org/genproto v0.0.0-20210909211513-a8c4777a87af h1:aLMMXFYqw01RA6XJim5uaN+afqNNjc9P8HPAbnpnc5s=
+google.golang.org/genproto v0.0.0-20210909211513-a8c4777a87af/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
 google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
 google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
@@ -476,9 +765,21 @@ google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyac
 google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
 google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
 google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
-google.golang.org/grpc v1.29.0/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk=
-google.golang.org/grpc v1.34.0 h1:raiipEjMOIC/TO2AvyTxP25XFdLxNIBwzDh3FM3XztI=
+google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60=
+google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk=
+google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
+google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
+google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
+google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0=
+google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc=
 google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8=
+google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
+google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
+google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
+google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM=
+google.golang.org/grpc v1.40.0 h1:AGJ0Ih4mHjSeibYkFGh1dD9KJ/eOtZ93I6hoHhukQ5Q=
+google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34=
 google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
 google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
 google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
@@ -488,8 +789,11 @@ google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2
 google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
 google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
 google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4=
-google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c=
 google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
+google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
+google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
+google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ=
+google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
@@ -500,6 +804,7 @@ gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
 gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
 gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
 gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
+gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
 gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=
 gopkg.in/sourcemap.v1 v1.0.5/go.mod h1:2RlvNNSMglmRrcvhfuzp4hQHwOtjxlbjX7UPY/GXb78=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
@@ -507,39 +812,45 @@ gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWD
 gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74=
 gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
-gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw=
+gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
+gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk=
+gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
-k8s.io/api v0.19.0/go.mod h1:I1K45XlvTrDjmj5LoM5LuP/KYrhWbjUKT/SoPG0qTjw=
-k8s.io/apimachinery v0.19.0/go.mod h1:DnPGDnARWFvYa3pMHgSxtbZb7gpzzAZ1pTfaUNDVlmA=
-k8s.io/client-go v0.19.0/go.mod h1:H9E/VT95blcFQnlyShFgnFT9ZnJOAceiUHM3MlRC+mU=
-k8s.io/component-base v0.19.0/go.mod h1:dKsY8BxkA+9dZIAh2aWJLL/UdASFDNtGYTCItL4LM7Y=
+honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
+honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
+k8s.io/api v0.22.0/go.mod h1:0AoXXqst47OI/L0oGKq9DG61dvGRPXs7X4/B7KyjBCU=
+k8s.io/apimachinery v0.22.0/go.mod h1:O3oNtNadZdeOMxHFVxOreoznohCpy0z6mocxbZr7oJ0=
+k8s.io/client-go v0.22.0/go.mod h1:GUjIuXR5PiEv/RVK5OODUsm6eZk7wtSWZSaSJbpFdGg=
+k8s.io/component-base v0.22.0/go.mod h1:SXj6Z+V6P6GsBhHZVbWCw9hFjUdUYnJerlhhPnYCBCg=
 k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0=
 k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE=
-k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y=
 k8s.io/klog/v2 v2.4.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y=
-k8s.io/klog/v2 v2.8.0 h1:Q3gmuM9hKEjefWFFYF0Mat+YyFJvsUyYuwyNNJ5C9Ts=
-k8s.io/klog/v2 v2.8.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec=
-k8s.io/kube-openapi v0.0.0-20200805222855-6aeccd4b50c6/go.mod h1:UuqjUnNftUyPE5H64/qeyjQoUZhGpeFDVdxjTeEVN2o=
-k8s.io/mount-utils v0.21.2 h1:tsCdw29Gv7PxwCQSeAmZ5WFkFLoNPdZVRouCnyDY/Lo=
-k8s.io/mount-utils v0.21.2/go.mod h1:dwXbIPxKtTjrBEaX1aK/CMEf1KZ8GzMHpe3NEBfdFXI=
-k8s.io/utils v0.0.0-20200729134348-d5654de09c73/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
-k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
-k8s.io/utils v0.0.0-20210527160623-6fdb442a123b h1:MSqsVQ3pZvPGTqCjptfimO2WjG7A9un2zcpiHkA6M/s=
-k8s.io/utils v0.0.0-20210527160623-6fdb442a123b/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
+k8s.io/klog/v2 v2.9.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec=
+k8s.io/klog/v2 v2.20.0 h1:tlyxlSvd63k7axjhuchckaRJm+a92z5GSOrTOQY5sHw=
+k8s.io/klog/v2 v2.20.0/go.mod h1:Gm8eSIfQN6457haJuPaMxZw4wyP5k+ykPFlrhQDvhvw=
+k8s.io/kube-openapi v0.0.0-20210421082810-95288971da7e/go.mod h1:vHXdDvt9+2spS2Rx9ql3I8tycm3H9FDfdUoIuKCefvw=
+k8s.io/mount-utils v0.22.1 h1:3xMxnOKobldBJMLAEpsC9Y3Drw1aaXz/fhTtYes4VTE=
+k8s.io/mount-utils v0.22.1/go.mod h1:gUi5ht+05KHYc/vJ9q9wbvG3MCYBeOsB5FdTyM60Pzo=
+k8s.io/utils v0.0.0-20210707171843-4b05e18ac7d9/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
+k8s.io/utils v0.0.0-20210820185131-d34e5cb4466e h1:ldQh+neBabomh7+89dTpiFAB8tGdfVmuIzAHbvtl+9I=
+k8s.io/utils v0.0.0-20210820185131-d34e5cb4466e/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
 rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
-sigs.k8s.io/structured-merge-diff/v4 v4.0.1/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw=
-sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o=
+rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
+rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
+sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw=
+sigs.k8s.io/structured-merge-diff/v4 v4.1.2/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4=
 sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc=

pkg/driver/controllerserver.go

@@ -19,15 +19,17 @@ package driver
 import (
 	"context"
 	"fmt"
+	log "github.com/sirupsen/logrus"
 	"time"
+	"sort"
 	"strconv"
+	"strings"
 
-	"github.com/golang/protobuf/ptypes"
+	"google.golang.org/protobuf/types/known/timestamppb"
 	"github.com/container-storage-interface/spec/lib/go/csi"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/status"
 
-	"github.com/SynologyOpenSource/synology-csi/pkg/dsm/webapi"
 	"github.com/SynologyOpenSource/synology-csi/pkg/interfaces"
 	"github.com/SynologyOpenSource/synology-csi/pkg/models"
 	"github.com/SynologyOpenSource/synology-csi/pkg/utils"
@@ -118,36 +120,56 @@ func (cs *controllerServer) CreateVolume(ctx context.Context, req *csi.CreateVol
 		isThin = utils.StringToBoolean(params["thin_provisioning"])
 	}
 
+	protocol := strings.ToLower(params["protocol"])
+	if protocol == "" {
+		protocol = utils.ProtocolDefault
+	} else if !isProtocolSupport(protocol) {
+		return nil, status.Error(codes.InvalidArgument, "Unsupported volume protocol")
+	}
+
 	spec := &models.CreateK8sVolumeSpec{
 		DsmIp:            params["dsm"],
 		K8sVolumeName:    volName,
-		LunName:          fmt.Sprintf("%s-%s", models.LunPrefix, volName),
+		LunName:          models.GenLunName(volName),
+		ShareName:        models.GenShareName(volName),
 		Location:         params["location"],
 		Size:             sizeInByte,
 		Type:             params["type"],
 		ThinProvisioning: isThin,
-		TargetName:       fmt.Sprintf("%s-%s", models.LunPrefix, volName),
+		TargetName:       fmt.Sprintf("%s-%s", models.TargetPrefix, volName),
 		MultipleSession:  multiSession,
 		SourceSnapshotId: srcSnapshotId,
 		SourceVolumeId:   srcVolumeId,
+		Protocol:         protocol,
 	}
 
-	lunInfo, dsmIp, err := cs.dsmService.CreateVolume(spec)
-	if err != nil {
-		return nil, err
+	// idempotency
+	// Note: an SMB PV may not be tested existed precisely because the share folder name was sliced from k8sVolumeName
+	k8sVolume := cs.dsmService.GetVolumeByName(volName)
+	if k8sVolume == nil {
+		k8sVolume, err = cs.dsmService.CreateVolume(spec)
+		if err != nil {
+			return nil, err
+		}
+	} else {
+		// already existed
+		log.Debugf("Volume [%s] already exists in [%s], backing name: [%s]", volName, k8sVolume.DsmIp, k8sVolume.Name)
 	}
 
-	if int64(lunInfo.Size) != sizeInByte {
+	if (k8sVolume.Protocol == utils.ProtocolIscsi && k8sVolume.SizeInBytes != sizeInByte) ||
+		(k8sVolume.Protocol == utils.ProtocolSmb && utils.BytesToMB(k8sVolume.SizeInBytes) != utils.BytesToMBCeil(sizeInByte)) {
 		return nil , status.Errorf(codes.AlreadyExists, "Already existing volume name with different capacity")
 	}
 
 	return &csi.CreateVolumeResponse{
 		Volume: &csi.Volume{
-			VolumeId: lunInfo.Uuid,
-			CapacityBytes: int64(lunInfo.Size),
+			VolumeId: k8sVolume.VolumeId,
+			CapacityBytes: k8sVolume.SizeInBytes,
 			ContentSource: volContentSrc,
 			VolumeContext: map[string]string{
-				"dsm": dsmIp,
+				"dsm":      k8sVolume.DsmIp,
+				"protocol": k8sVolume.Protocol,
+				"source":   k8sVolume.Source,
 			},
 		},
 	}, nil
@@ -212,9 +234,11 @@ func (cs *controllerServer) ListVolumes(ctx context.Context, req *csi.ListVolume
 	pagingSkip := ("" != startingToken)
 	infos := cs.dsmService.ListVolumes()
 
+	sort.Sort(models.ByVolumeId(infos))
+
 	var count int32 = 0
 	for _, info := range infos {
-		if info.Lun.Uuid == startingToken {
+		if info.VolumeId == startingToken {
 			pagingSkip = false
 		}
 
@@ -223,18 +247,20 @@ func (cs *controllerServer) ListVolumes(ctx context.Context, req *csi.ListVolume
 		}
 
 		if maxEntries > 0 && count >= maxEntries {
-			nextToken = info.Lun.Uuid
+			nextToken = info.VolumeId
 			break
 		}
 
 		entries = append(entries, &csi.ListVolumesResponse_Entry{
 			Volume: &csi.Volume{
-				VolumeId:      info.Lun.Uuid,
-				CapacityBytes: int64(info.Lun.Size),
+				VolumeId:      info.VolumeId,
+				CapacityBytes: info.SizeInBytes,
 				VolumeContext: map[string]string{
 					"dsm":       info.DsmIp,
 					"lunName":   info.Lun.Name,
 					"targetIqn": info.Target.Iqn,
+					"shareName": info.Share.Name,
+					"protocol":  info.Protocol,
 				},
 			},
 		})
@@ -290,7 +316,7 @@ func (cs *controllerServer) ControllerGetCapabilities(ctx context.Context, req *
 
 func (cs *controllerServer) CreateSnapshot(ctx context.Context, req *csi.CreateSnapshotRequest) (*csi.CreateSnapshotResponse, error) {
 	srcVolId := req.GetSourceVolumeId()
-	snapshotName := req.GetName()
+	snapshotName := req.GetName() // snapshot-XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX
 	params := req.GetParameters()
 
 	if srcVolId == "" {
@@ -301,37 +327,28 @@ func (cs *controllerServer) CreateSnapshot(ctx context.Context, req *csi.CreateS
 		return nil, status.Error(codes.InvalidArgument, "Snapshot name is empty.")
 	}
 
-	snapshotInfos, err := cs.dsmService.ListAllSnapshots()
-
-	if err != nil {
-		return nil, status.Errorf(codes.Internal, fmt.Sprintf("Failed to ListAllSnapshots(), err: %v", err))
-	}
-
 	// idempotency
-	for _, snapshotInfo := range snapshotInfos {
-		if snapshotInfo.Name == snapshotName {
-			if snapshotInfo.ParentUuid != srcVolId {
-				return nil, status.Errorf(codes.AlreadyExists, fmt.Sprintf("Snapshot [%s] already exists but volume id is incompatible", snapshotName))
-			}
-
-			createTime, err := ptypes.TimestampProto(time.Unix(snapshotInfo.CreateTime, 0))
-
-			if err != nil {
-				return nil, status.Errorf(codes.Internal, fmt.Sprintf("Failed to convert create time, err: %v", err))
-			}
-
-			return &csi.CreateSnapshotResponse{
-				Snapshot: &csi.Snapshot{
-					SizeBytes: snapshotInfo.TotalSize,
-					SnapshotId: snapshotInfo.Uuid,
-					SourceVolumeId: snapshotInfo.ParentUuid,
-					CreationTime: createTime,
-					ReadyToUse: (snapshotInfo.Status == "Healthy"),
-				},
-			}, nil
+	orgSnap := cs.dsmService.GetSnapshotByName(snapshotName)
+	if orgSnap != nil {
+		// already existed
+		if orgSnap.ParentUuid != srcVolId {
+			return nil, status.Errorf(codes.AlreadyExists, fmt.Sprintf("Snapshot [%s] already exists but volume id is incompatible", snapshotName))
 		}
+		if orgSnap.CreateTime < 0 {
+			return nil, status.Errorf(codes.Internal, fmt.Sprintf("Bad create time: %v", orgSnap.CreateTime))
+		}
+		return &csi.CreateSnapshotResponse{
+			Snapshot: &csi.Snapshot{
+				SizeBytes: orgSnap.SizeInBytes,
+				SnapshotId: orgSnap.Uuid,
+				SourceVolumeId: orgSnap.ParentUuid,
+				CreationTime: timestamppb.New(time.Unix(orgSnap.CreateTime, 0)),
+				ReadyToUse: (orgSnap.Status == "Healthy"),
+			},
+		}, nil
 	}
 
+	// not exist, going to create a new snapshot
 	spec := &models.CreateK8sVolumeSnapshotSpec{
 		K8sVolumeId:  srcVolId,
 		SnapshotName: snapshotName,
@@ -340,35 +357,19 @@ func (cs *controllerServer) CreateSnapshot(ctx context.Context, req *csi.CreateS
 		IsLocked:     utils.StringToBoolean(params["is_locked"]),
 	}
 
-	snapshotId, err := cs.dsmService.CreateSnapshot(spec)
-
+	snapshot, err := cs.dsmService.CreateSnapshot(spec)
 	if err != nil {
-		if err == utils.OutOfFreeSpaceError("") || err == utils.SnapshotReachMaxCountError("") {
-			return nil,status.Errorf(codes.ResourceExhausted, fmt.Sprintf("Failed to CreateSnapshot(%s), err: %v", srcVolId, err))
-		} else {
-			return nil, status.Errorf(codes.Internal, fmt.Sprintf("Failed to CreateSnapshot(%s), err: %v", srcVolId, err))
-		}
-	}
-
-	snapshotInfo, err := cs.dsmService.GetSnapshot(snapshotId)
-
-	if err != nil {
-		return nil, status.Errorf(codes.Internal, fmt.Sprintf("Failed to GetSnapshot(%s), err: %v", snapshotId, err))
-	}
-
-	createTime, err := ptypes.TimestampProto(time.Unix(snapshotInfo.CreateTime, 0))
-
-	if err != nil {
-		return nil, status.Errorf(codes.Internal, fmt.Sprintf("Failed to convert create time, err: %v", err))
+		log.Errorf("Failed to CreateSnapshot, snapshotName: %s, srcVolId: %s, err: %v", snapshotName, srcVolId, err)
+		return nil, err
 	}
 
 	return &csi.CreateSnapshotResponse{
 		Snapshot: &csi.Snapshot{
-			SizeBytes: snapshotInfo.TotalSize,
-			SnapshotId: snapshotInfo.Uuid,
-			SourceVolumeId: snapshotInfo.ParentUuid,
-			CreationTime: createTime,
-			ReadyToUse: (snapshotInfo.Status == "Healthy"),
+			SizeBytes: snapshot.SizeInBytes,
+			SnapshotId: snapshot.Uuid,
+			SourceVolumeId: snapshot.ParentUuid,
+			CreationTime: timestamppb.New(time.Unix(snapshot.CreateTime, 0)),
+			ReadyToUse: (snapshot.Status == "Healthy"),
 		},
 	}, nil
 }
@@ -402,22 +403,19 @@ func (cs *controllerServer) ListSnapshots(ctx context.Context, req *csi.ListSnap
 	}
 
 	pagingSkip := ("" != startingToken)
-	var snapshotInfos []webapi.SnapshotInfo
-	var err error
+	var snapshots []*models.K8sSnapshotRespSpec
 
 	if (srcVolId != "") {
-		snapshotInfos, err = cs.dsmService.ListSnapshots(srcVolId)
+		snapshots = cs.dsmService.ListSnapshots(srcVolId)
 	} else {
-		snapshotInfos, err = cs.dsmService.ListAllSnapshots()
+		snapshots = cs.dsmService.ListAllSnapshots()
 	}
 
-	if err != nil {
-		return nil, status.Errorf(codes.Internal, fmt.Sprintf("Failed to ListSnapshots(%s), err: %v", srcVolId, err))
-	}
+	sort.Sort(models.BySnapshotAndParentUuid(snapshots))
 
 	var count int32 = 0
-	for _, snapshotInfo := range snapshotInfos {
-		if snapshotInfo.Uuid == startingToken {
+	for _, snapshot := range snapshots {
+		if snapshot.Uuid == startingToken {
 			pagingSkip = false
 		}
 
@@ -425,28 +423,21 @@ func (cs *controllerServer) ListSnapshots(ctx context.Context, req *csi.ListSnap
 			continue
 		}
 
-		if snapshotId != "" && snapshotInfo.Uuid != snapshotId {
+		if snapshotId != "" && snapshot.Uuid != snapshotId {
 			continue
 		}
 
 		if maxEntries > 0 && count >= maxEntries {
-			nextToken = snapshotInfo.Uuid
+			nextToken = snapshot.Uuid
 			break
 		}
-
-		createTime, err := ptypes.TimestampProto(time.Unix(snapshotInfo.CreateTime, 0))
-
-		if err != nil {
-			return nil, status.Errorf(codes.Internal, fmt.Sprintf("Failed to convert create time, err: %v", err))
-		}
-
 		entries = append(entries, &csi.ListSnapshotsResponse_Entry{
 			Snapshot: &csi.Snapshot{
-				SizeBytes: snapshotInfo.TotalSize,
-				SnapshotId: snapshotInfo.Uuid,
-				SourceVolumeId: snapshotInfo.ParentUuid,
-				CreationTime: createTime,
-				ReadyToUse: (snapshotInfo.Status == "Healthy"),
+				SizeBytes: snapshot.SizeInBytes,
+				SnapshotId: snapshot.Uuid,
+				SourceVolumeId: snapshot.ParentUuid,
+				CreationTime: timestamppb.New(time.Unix(snapshot.CreateTime, 0)),
+				ReadyToUse: (snapshot.Status == "Healthy"),
 			},
 		})
 
@@ -477,14 +468,14 @@ func (cs *controllerServer) ControllerExpandVolume(ctx context.Context, req *csi
 			"InvalidArgument: Please check CapacityRange[%v]", capRange)
 	}
 
-	if err := cs.dsmService.ExpandLun(volumeId, sizeInByte); err != nil {
-		return nil, status.Error(codes.Internal,
-			fmt.Sprintf("Failed to expand volume [%s], err: %v", volumeId, err))
+	k8sVolume, err := cs.dsmService.ExpandVolume(volumeId, sizeInByte)
+	if err != nil {
+		return nil, err
 	}
 
 	return &csi.ControllerExpandVolumeResponse{
-		CapacityBytes: sizeInByte,
-		NodeExpansionRequired: true,
+		CapacityBytes: k8sVolume.SizeInBytes,
+		NodeExpansionRequired: (k8sVolume.Protocol == utils.ProtocolIscsi),
 	}, nil
 }
 

pkg/driver/driver.go

@@ -20,11 +20,16 @@ import (
 	"github.com/container-storage-interface/spec/lib/go/csi"
 	log "github.com/sirupsen/logrus"
 	"github.com/SynologyOpenSource/synology-csi/pkg/interfaces"
+	"github.com/SynologyOpenSource/synology-csi/pkg/utils"
 )
 
 const (
 	DriverName = "csi.san.synology.com" // CSI dirver name
-	DriverVersion = "1.0.0"
+	DriverVersion = "1.1.0"
+)
+
+var (
+	supportedProtocolList = []string{utils.ProtocolIscsi, utils.ProtocolSmb}
 )
 
 type IDriver interface {
@@ -73,6 +78,8 @@ func NewControllerAndNodeDriver(nodeID string, endpoint string, dsmService inter
 	d.addNodeServiceCapabilities([]csi.NodeServiceCapability_RPC_Type{
 		csi.NodeServiceCapability_RPC_STAGE_UNSTAGE_VOLUME,
 		csi.NodeServiceCapability_RPC_EXPAND_VOLUME,
+		csi.NodeServiceCapability_RPC_VOLUME_MOUNT_GROUP,
+		// csi.NodeServiceCapability_RPC_GET_VOLUME_STATS, //TODO
 	})
 
 	log.Infof("New driver created: name=%s, nodeID=%s, version=%s, endpoint=%s", d.name, d.nodeID, d.version, d.endpoint)
@@ -127,3 +134,7 @@ func (d *Driver) addNodeServiceCapabilities(nsc []csi.NodeServiceCapability_RPC_
 func (d *Driver) getVolumeCapabilityAccessModes() []*csi.VolumeCapability_AccessMode { // for debugging
 	return d.vCap
 }
+
+func isProtocolSupport(protocol string) bool {
+	return utils.SliceContains(supportedProtocolList, protocol)
+}

pkg/driver/nodeserver.go

@@ -23,19 +23,22 @@ import (
 	"strings"
 	"time"
 
+	"github.com/cenkalti/backoff/v4"
 	"github.com/container-storage-interface/spec/lib/go/csi"
 	log "github.com/sirupsen/logrus"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/status"
 	"k8s.io/mount-utils"
 
+	"github.com/SynologyOpenSource/synology-csi/pkg/dsm/webapi"
 	"github.com/SynologyOpenSource/synology-csi/pkg/interfaces"
+	"github.com/SynologyOpenSource/synology-csi/pkg/models"
 	"github.com/SynologyOpenSource/synology-csi/pkg/utils"
 )
 
 type nodeServer struct {
-	Driver *Driver
-	Mounter *mount.SafeFormatAndMount
+	Driver     *Driver
+	Mounter    *mount.SafeFormatAndMount
 	dsmService interfaces.IDsmService
 	Initiator  *initiatorDriver
 }
@@ -122,7 +125,7 @@ func createTargetMountPath(mounter mount.Interface, mountPath string, isBlock bo
 }
 
 func (ns *nodeServer) loginTarget(volumeId string) error {
-	k8sVolume := ns.dsmService.GetVolume(volumeId);
+	k8sVolume := ns.dsmService.GetVolume(volumeId)
 
 	if k8sVolume == nil {
 		return status.Error(codes.NotFound, fmt.Sprintf("Volume[%s] is not found", volumeId))
@@ -139,13 +142,182 @@ func (ns *nodeServer) loginTarget(volumeId string) error {
 func (ns *nodeServer) logoutTarget(volumeId string) {
 	k8sVolume := ns.dsmService.GetVolume(volumeId)
 
-	if k8sVolume == nil {
+	if k8sVolume == nil || k8sVolume.Protocol != utils.ProtocolIscsi {
 		return
 	}
 
 	ns.Initiator.logout(k8sVolume.Target.Iqn, k8sVolume.DsmIp)
 }
 
+func checkGidPresentInMountFlags(volumeMountGroup string, mountFlags []string) (bool, error) {
+	gidPresentInMountFlags := false
+	for _, mountFlag := range mountFlags {
+		if strings.HasPrefix(mountFlag, "gid") {
+			gidPresentInMountFlags = true
+			kvpair := strings.Split(mountFlag, "=")
+			if volumeMountGroup != "" && len(kvpair) == 2 && !strings.EqualFold(volumeMountGroup, kvpair[1]) {
+				return false, status.Error(codes.InvalidArgument, fmt.Sprintf("gid(%s) in storageClass and pod fsgroup(%s) are not equal", kvpair[1], volumeMountGroup))
+			}
+		}
+	}
+	return gidPresentInMountFlags, nil
+}
+
+func (ns *nodeServer) mountSensitiveWithRetry(sourcePath string, targetPath string, fsType string, options []string, sensitiveOptions []string) error {
+	mountBackoff := backoff.NewExponentialBackOff()
+	mountBackoff.InitialInterval = 1 * time.Second
+	mountBackoff.Multiplier = 2
+	mountBackoff.RandomizationFactor = 0.1
+	mountBackoff.MaxElapsedTime = 5 * time.Second
+
+	checkFinished := func() error {
+		if err := ns.Mounter.MountSensitive(sourcePath, targetPath, fsType, options, sensitiveOptions); err != nil {
+			return err
+		}
+
+		return nil
+	}
+
+	mountNotify := func(err error, duration time.Duration) {
+		log.Infof("Retry MountSensitive, waiting %3.2f seconds .....", float64(duration.Seconds()))
+	}
+
+	if err := backoff.RetryNotify(checkFinished, mountBackoff, mountNotify); err != nil {
+		log.Errorf("Could not finish mount after %3.2f seconds.", float64(mountBackoff.MaxElapsedTime.Seconds()))
+		return err
+	}
+
+	log.Debugf("Mount successfully. source: %s, target: %s", sourcePath, targetPath)
+	return nil
+}
+
+func (ns *nodeServer) setSMBVolumePermission(sourcePath string, userName string, authType utils.AuthType) error {
+	s := strings.Split(strings.TrimPrefix(sourcePath, "//"), "/")
+	if len(s) != 2 {
+		return fmt.Errorf("Failed to parse dsmIp and shareName from source path")
+	}
+	dsmIp, shareName := s[0], s[1]
+
+	dsm, err := ns.dsmService.GetDsm(dsmIp)
+	if err != nil {
+		return fmt.Errorf("Failed to get DSM[%s]", dsmIp)
+	}
+
+	permission := webapi.SharePermission{
+		Name: userName,
+	}
+	switch authType {
+	case utils.AuthTypeReadWrite:
+		permission.IsWritable = true
+	case utils.AuthTypeReadOnly:
+		permission.IsReadonly = true
+	case utils.AuthTypeNoAccess:
+		permission.IsDeny = true
+	default:
+		return fmt.Errorf("Unknown auth type: %s", string(authType))
+	}
+
+	permissions := append([]*webapi.SharePermission{}, &permission)
+	spec := webapi.SharePermissionSetSpec{
+		Name: shareName,
+		UserGroupType: models.UserGroupTypeLocalUser,
+		Permissions: permissions,
+	}
+
+	return dsm.SharePermissionSet(spec)
+}
+
+func (ns *nodeServer) nodeStageISCSIVolume(ctx context.Context, spec *models.NodeStageVolumeSpec) (*csi.NodeStageVolumeResponse, error) {
+	// if block mode, skip mount
+	if spec.VolumeCapability.GetBlock() != nil {
+		return &csi.NodeStageVolumeResponse{}, nil
+	}
+
+	if err := ns.loginTarget(spec.VolumeId); err != nil {
+		return nil, status.Error(codes.Internal, err.Error())
+	}
+
+	volumeMountPath := ns.getVolumeMountPath(spec.VolumeId)
+	if volumeMountPath == "" {
+		return nil, status.Error(codes.Internal, "Can't get volume mount path")
+	}
+
+	notMount, err := ns.Mounter.Interface.IsLikelyNotMountPoint(spec.StagingTargetPath)
+	if err != nil {
+		return nil, status.Error(codes.Internal, err.Error())
+	}
+
+	if !notMount {
+		return &csi.NodeStageVolumeResponse{}, nil
+	}
+
+	fsType := spec.VolumeCapability.GetMount().GetFsType()
+	options := append([]string{"rw"}, spec.VolumeCapability.GetMount().GetMountFlags()...)
+
+	if err = ns.Mounter.FormatAndMount(volumeMountPath, spec.StagingTargetPath, fsType, options); err != nil {
+		return nil, status.Error(codes.Internal, err.Error())
+	}
+
+	return &csi.NodeStageVolumeResponse{}, nil
+}
+
+func (ns *nodeServer) nodeStageSMBVolume(ctx context.Context, spec *models.NodeStageVolumeSpec, secrets map[string]string) (*csi.NodeStageVolumeResponse, error) {
+	if spec.VolumeCapability.GetBlock() != nil {
+		return nil, status.Error(codes.InvalidArgument, fmt.Sprintf("SMB protocol only allows 'mount' access type"))
+	}
+
+	if spec.Source == "" { //"//<host>/<shareName>"
+		return nil, status.Error(codes.InvalidArgument, fmt.Sprintf("Missing 'source' field"))
+	}
+
+	if secrets == nil {
+		return nil, status.Error(codes.InvalidArgument, fmt.Sprintf("Missing secrets for node staging volume"))
+	}
+
+	username := strings.TrimSpace(secrets["username"])
+	password := strings.TrimSpace(secrets["password"])
+	domain := strings.TrimSpace(secrets["domain"])
+
+	// set permission to access the share
+	if err := ns.setSMBVolumePermission(spec.Source, username, utils.AuthTypeReadWrite); err != nil {
+		return nil, status.Error(codes.Internal, fmt.Sprintf("Failed to set permission, source: %s, err: %v", spec.Source, err))
+	}
+
+	// create mount point if not exists
+	targetPath := spec.StagingTargetPath
+	notMount, err := createTargetMountPath(ns.Mounter.Interface, targetPath, false)
+	if err != nil {
+		return nil, status.Error(codes.Internal, err.Error())
+	}
+	if !notMount {
+		log.Infof("NodeStageVolume: %s is already mounted", targetPath)
+		return &csi.NodeStageVolumeResponse{}, nil // already mount
+	}
+
+	fsType := "cifs"
+	options := spec.VolumeCapability.GetMount().GetMountFlags()
+
+	volumeMountGroup := spec.VolumeCapability.GetMount().GetVolumeMountGroup()
+	gidPresent, err := checkGidPresentInMountFlags(volumeMountGroup, options)
+	if err != nil {
+		return nil, err
+	}
+	if !gidPresent && volumeMountGroup != "" {
+		options = append(options, fmt.Sprintf("gid=%s", volumeMountGroup))
+	}
+
+
+	if domain != "" {
+		options = append(options, fmt.Sprintf("%s=%s", "domain", domain))
+	}
+	var sensitiveOptions = []string{fmt.Sprintf("%s=%s,%s=%s", "username", username, "password", password)}
+	if err := ns.mountSensitiveWithRetry(spec.Source, targetPath, fsType, options, sensitiveOptions); err != nil {
+		return nil, status.Error(codes.Internal,
+			fmt.Sprintf("Volume[%s] failed to mount %q on %q. err: %v", spec.VolumeId, spec.Source, targetPath, err))
+	}
+	return &csi.NodeStageVolumeResponse{}, nil
+}
+
 func (ns *nodeServer) NodeStageVolume(ctx context.Context, req *csi.NodeStageVolumeRequest) (*csi.NodeStageVolumeResponse, error) {
 	volumeId, stagingTargetPath, volumeCapability :=
 		req.GetVolumeId(), req.GetStagingTargetPath(), req.GetVolumeCapability()
@@ -155,47 +327,32 @@ func (ns *nodeServer) NodeStageVolume(ctx context.Context, req *csi.NodeStageVol
 		 "InvalidArgument: Please check volume ID, staging target path and volume capability.")
 	}
 
-	// if block mode, skip mount
-	if volumeCapability.GetBlock() != nil {
-		return &csi.NodeStageVolumeResponse{}, nil
+	if volumeCapability.GetBlock() != nil && volumeCapability.GetMount() != nil {
+		return nil, status.Error(codes.InvalidArgument, "Cannot mix block and mount capabilities")
 	}
 
-	if err := ns.loginTarget(volumeId); err != nil {
-		return nil, status.Error(codes.Internal, err.Error())
+	spec := &models.NodeStageVolumeSpec{
+		VolumeId: volumeId,
+		StagingTargetPath: stagingTargetPath,
+		VolumeCapability: volumeCapability,
+		Dsm: req.VolumeContext["dsm"],
+		Source: req.VolumeContext["source"], // filled by CreateVolume response
 	}
 
-	volumeMountPath := ns.getVolumeMountPath(volumeId)
-	if volumeMountPath == "" {
-		return nil, status.Error(codes.Internal, "Can't get volume mount path")
+	switch req.VolumeContext["protocol"] {
+	case utils.ProtocolSmb:
+		return ns.nodeStageSMBVolume(ctx, spec, req.GetSecrets())
+	default:
+		return ns.nodeStageISCSIVolume(ctx, spec)
 	}
-
-	notMount, err := ns.Mounter.Interface.IsLikelyNotMountPoint(stagingTargetPath)
-	if err != nil {
-		return nil, status.Error(codes.Internal, err.Error())
-	}
-
-	if !notMount {
-		return &csi.NodeStageVolumeResponse{}, nil
-	}
-
-	fsType := volumeCapability.GetMount().GetFsType()
-	mountFlags := volumeCapability.GetMount().GetMountFlags()
-	options := append([]string{"rw"}, mountFlags...)
-
-	if err = ns.Mounter.FormatAndMount(volumeMountPath, stagingTargetPath, fsType, options); err != nil {
-		return nil, status.Error(codes.Internal, err.Error())
-	}
-
-	return &csi.NodeStageVolumeResponse{}, nil
 }
 
 func (ns *nodeServer) NodeUnstageVolume(ctx context.Context, req *csi.NodeUnstageVolumeRequest) (*csi.NodeUnstageVolumeResponse, error) {
-	if req.GetVolumeId() == "" { // Useless, just for sanity check
+	volumeID, stagingTargetPath := req.GetVolumeId(), req.GetStagingTargetPath()
+
+	if volumeID == "" {
 		return nil, status.Error(codes.InvalidArgument, "Volume ID missing in request")
 	}
-
-	stagingTargetPath := req.GetStagingTargetPath()
-
 	if stagingTargetPath == "" {
 		return nil, status.Error(codes.InvalidArgument, "Target path missing in request")
 	}
@@ -211,18 +368,26 @@ func (ns *nodeServer) NodeUnstageVolume(ctx context.Context, req *csi.NodeUnstag
 		}
 	}
 
+	ns.logoutTarget(volumeID)
+
 	return &csi.NodeUnstageVolumeResponse{}, nil
 }
 
 func (ns *nodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublishVolumeRequest) (*csi.NodePublishVolumeResponse, error) {
 	volumeId, targetPath, stagingTargetPath := req.GetVolumeId(), req.GetTargetPath(), req.GetStagingTargetPath()
-	isBlock := req.GetVolumeCapability().GetBlock() != nil
 
 	if volumeId == "" || targetPath == "" || stagingTargetPath == "" {
 		return nil, status.Error(codes.InvalidArgument,
 			"InvalidArgument: Please check volume ID, target path and staging target path.")
 	}
 
+	if req.GetVolumeCapability() == nil {
+		return nil, status.Error(codes.InvalidArgument, "Volume capability missing in request")
+	}
+
+	isBlock := req.GetVolumeCapability().GetBlock() != nil // raw block, only for iscsi protocol
+	fsType := req.GetVolumeCapability().GetMount().GetFsType()
+
 	notMount, err := createTargetMountPath(ns.Mounter.Interface, targetPath, isBlock)
 	if err != nil {
 		return nil, status.Error(codes.Internal, err.Error())
@@ -231,45 +396,51 @@ func (ns *nodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublis
 		return &csi.NodePublishVolumeResponse{}, nil
 	}
 
-	if err := ns.loginTarget(volumeId); err != nil {
-		return nil, status.Error(codes.Internal, err.Error())
-	}
-
-	volumeMountPath := ns.getVolumeMountPath(volumeId)
-	if volumeMountPath == "" {
-		return nil, status.Error(codes.Internal, "Can't get volume mount path")
-	}
-
 	options := []string{"bind"}
 	if req.GetReadonly() {
 		options = append(options, "ro")
 	}
 
-	if isBlock {
-		err = ns.Mounter.Interface.Mount(volumeMountPath, targetPath, "", options)
-	} else {
-		fsType := req.GetVolumeCapability().GetMount().GetFsType()
-		err = ns.Mounter.Interface.Mount(stagingTargetPath, targetPath, fsType, options)
-	}
+	switch req.VolumeContext["protocol"] {
+	case utils.ProtocolSmb:
+		if err := ns.Mounter.Interface.Mount(stagingTargetPath, targetPath, "", options); err != nil {
+			return nil, status.Error(codes.Internal, err.Error())
+		}
+	default:
+		if err := ns.loginTarget(volumeId); err != nil {
+			return nil, status.Error(codes.Internal, err.Error())
+		}
 
-	if err != nil {
-		return nil, status.Error(codes.Internal, err.Error())
+		volumeMountPath := ns.getVolumeMountPath(volumeId)
+		if volumeMountPath == "" {
+			return nil, status.Error(codes.Internal, "Can't get volume mount path")
+		}
+
+		if isBlock {
+			err = ns.Mounter.Interface.Mount(volumeMountPath, targetPath, "", options)
+		} else {
+			err = ns.Mounter.Interface.Mount(stagingTargetPath, targetPath, fsType, options)
+		}
+		if err != nil {
+			return nil, status.Error(codes.Internal, err.Error())
+		}
 	}
 
 	return &csi.NodePublishVolumeResponse{}, nil
 }
 
 func (ns *nodeServer) NodeUnpublishVolume(ctx context.Context, req *csi.NodeUnpublishVolumeRequest) (*csi.NodeUnpublishVolumeResponse, error) {
-	volumeId, targetPath := req.GetVolumeId(), req.GetTargetPath()
-	if volumeId == "" {
+	if req.GetVolumeId() == "" { // Not needed, but still a mandatory field
 		return nil, status.Error(codes.InvalidArgument, "Volume ID missing in request")
 	}
+
+	targetPath := req.GetTargetPath()
 	if targetPath == "" {
 		return nil, status.Error(codes.InvalidArgument, "Target path missing in request")
 	}
 
 	if _, err := os.Stat(targetPath); err != nil {
-		if os.IsNotExist(err){
+		if os.IsNotExist(err) {
 			return &csi.NodeUnpublishVolumeResponse{}, nil
 		}
 		return nil, status.Errorf(codes.Internal, err.Error())
@@ -285,24 +456,6 @@ func (ns *nodeServer) NodeUnpublishVolume(ctx context.Context, req *csi.NodeUnpu
 		return &csi.NodeUnpublishVolumeResponse{}, nil
 	}
 
-	needToLogout := true
-
-	list, err := ns.Mounter.Interface.GetMountRefs(targetPath)
-
-	if err != nil {
-		return nil, status.Error(codes.Internal, err.Error())
-	}
-
-	for _, path := range list {
-		filePrefix := "/var/lib/kubelet/pods/"
-		blkPrefix := "/var/lib/kubelet/plugins/kubernetes.io/csi/volumeDevices/publish/"
-
-		if strings.HasPrefix(path, filePrefix) || strings.HasPrefix(path, blkPrefix) {
-			needToLogout = false
-			break
-		}
-	}
-
 	if err := ns.Mounter.Interface.Unmount(targetPath); err != nil {
 		return nil, status.Error(codes.Internal, err.Error())
 	}
@@ -311,10 +464,6 @@ func (ns *nodeServer) NodeUnpublishVolume(ctx context.Context, req *csi.NodeUnpu
 		return nil, status.Errorf(codes.Internal, "Failed to remove target path.")
 	}
 
-	if needToLogout {
-		ns.logoutTarget(volumeId)
-	}
-
 	return &csi.NodeUnpublishVolumeResponse{}, nil
 }
 
@@ -350,8 +499,19 @@ func (ns *nodeServer) NodeGetVolumeStats(ctx context.Context, req *csi.NodeGetVo
 			fmt.Sprintf("Volume[%s] does not exist on the %s", volumeId, volumePath))
 	}
 
-	lun := k8sVolume.Lun
 
+	if k8sVolume.Protocol == utils.ProtocolSmb {
+		return &csi.NodeGetVolumeStatsResponse{
+			Usage: []*csi.VolumeUsage{
+				&csi.VolumeUsage{
+					Total:     k8sVolume.SizeInBytes,
+					Unit:      csi.VolumeUsage_BYTES,
+				},
+			},
+		}, nil
+	}
+
+	lun := k8sVolume.Lun
 	return &csi.NodeGetVolumeStatsResponse{
 		Usage: []*csi.VolumeUsage{
 			&csi.VolumeUsage{
@@ -376,6 +536,11 @@ func (ns *nodeServer) NodeExpandVolume(ctx context.Context, req *csi.NodeExpandV
 		return nil, status.Error(codes.NotFound, fmt.Sprintf("Volume[%s] is not found", volumeId))
 	}
 
+	if k8sVolume.Protocol == utils.ProtocolSmb {
+		return &csi.NodeExpandVolumeResponse{
+			CapacityBytes: sizeInByte}, nil
+	}
+
 	if err := ns.Initiator.rescan(k8sVolume.Target.Iqn); err != nil {
 		return nil, status.Error(codes.Internal, fmt.Sprintf("Failed to rescan. err: %v", err))
 	}
@@ -400,4 +565,4 @@ func (ns *nodeServer) NodeExpandVolume(ctx context.Context, req *csi.NodeExpandV
 	}
 	return &csi.NodeExpandVolumeResponse{
 		CapacityBytes: sizeInByte}, nil
-}
+}

pkg/dsm/service/dsm.go

@@ -149,11 +149,11 @@ func getLunTypeByInputParams(lunType string, isThin bool, locationFsType string)
 	return "", fmt.Errorf("Unknown volume fs type: %s", locationFsType)
 }
 
-func (service *DsmService) createMappingTarget(dsm *webapi.DSM, spec *models.CreateK8sVolumeSpec, lunUuid string) error {
+func (service *DsmService) createMappingTarget(dsm *webapi.DSM, spec *models.CreateK8sVolumeSpec, lunUuid string) (webapi.TargetInfo, error) {
 	dsmInfo, err := dsm.DsmInfoGet()
 
 	if err != nil {
-		return status.Errorf(codes.Internal, fmt.Sprintf("Failed to get DSM[%s] info", dsm.Ip));
+		return webapi.TargetInfo{}, status.Errorf(codes.Internal, fmt.Sprintf("Failed to get DSM[%s] info", dsm.Ip));
 	}
 
 	genTargetIqn := func() string {
@@ -175,35 +175,35 @@ func (service *DsmService) createMappingTarget(dsm *webapi.DSM, spec *models.Cre
 	targetId, err := dsm.TargetCreate(targetSpec)
 
 	if err != nil && !errors.Is(err, utils.AlreadyExistError("")) {
-		return status.Errorf(codes.Internal, fmt.Sprintf("Failed to create target with spec: %v, err: %v", targetSpec, err))
+		return webapi.TargetInfo{}, status.Errorf(codes.Internal, fmt.Sprintf("Failed to create target with spec: %v, err: %v", targetSpec, err))
 	}
 
-	if targetInfo, err := dsm.TargetGet(targetSpec.Name); err != nil {
-		return status.Errorf(codes.Internal, fmt.Sprintf("Failed to get target with spec: %v, err: %v", targetSpec, err))
+	targetInfo, err := dsm.TargetGet(targetSpec.Name)
+	if err != nil {
+		return webapi.TargetInfo{}, status.Errorf(codes.Internal, fmt.Sprintf("Failed to get target with spec: %v, err: %v", targetSpec, err))
 	} else {
 		targetId = strconv.Itoa(targetInfo.TargetId);
 	}
 
 	if spec.MultipleSession == true {
 		if err := dsm.TargetSet(targetId, 0); err != nil {
-			return status.Errorf(codes.Internal, fmt.Sprintf("Failed to set target [%s] max session, err: %v", spec.TargetName, err))
+			return webapi.TargetInfo{}, status.Errorf(codes.Internal, fmt.Sprintf("Failed to set target [%s] max session, err: %v", spec.TargetName, err))
 		}
 	}
 
-	err = dsm.LunMapTarget([]string{targetId}, lunUuid)
-	if err != nil {
-		return status.Errorf(codes.Internal, fmt.Sprintf("Failed to map target [%s] to lun [%s], err: %v", spec.TargetName, lunUuid, err))
+	if err := dsm.LunMapTarget([]string{targetId}, lunUuid); err != nil {
+		return webapi.TargetInfo{}, status.Errorf(codes.Internal, fmt.Sprintf("Failed to map target [%s] to lun [%s], err: %v", spec.TargetName, lunUuid, err))
 	}
 
-	return nil
+	return targetInfo, nil
 }
 
-func (service *DsmService) createVolumeByDsm(dsm *webapi.DSM, spec *models.CreateK8sVolumeSpec) (webapi.LunInfo, error) {
+func (service *DsmService) createVolumeByDsm(dsm *webapi.DSM, spec *models.CreateK8sVolumeSpec) (*models.K8sVolumeRespSpec, error) {
 	// 1. Find a available location
 	if spec.Location == "" {
 		vol, err := service.getFirstAvailableVolume(dsm, spec.Size)
 		if err != nil {
-			return webapi.LunInfo{},
+			return nil,
 				status.Errorf(codes.Internal, fmt.Sprintf("Failed to get available location, err: %v", err))
 		}
 		spec.Location = vol.Path
@@ -212,13 +212,13 @@ func (service *DsmService) createVolumeByDsm(dsm *webapi.DSM, spec *models.Creat
 	// 2. Check if location exists
 	dsmVolInfo, err := dsm.VolumeGet(spec.Location)
 	if err != nil {
-		return webapi.LunInfo{},
+		return nil,
 			status.Errorf(codes.InvalidArgument, fmt.Sprintf("Unable to find location %s", spec.Location))
 	}
 
 	lunType, err := getLunTypeByInputParams(spec.Type, spec.ThinProvisioning, dsmVolInfo.FsType)
 	if err != nil {
-		return webapi.LunInfo{},
+		return nil,
 			status.Errorf(codes.InvalidArgument, fmt.Sprintf("Unknown volume fs type: %s, location: %s", dsmVolInfo.FsType, spec.Location))
 	}
 
@@ -234,29 +234,29 @@ func (service *DsmService) createVolumeByDsm(dsm *webapi.DSM, spec *models.Creat
 	_, err = dsm.LunCreate(lunSpec)
 
 	if err != nil && !errors.Is(err, utils.AlreadyExistError("")) {
-		return webapi.LunInfo{},
-			status.Errorf(codes.Internal, fmt.Sprintf("Failed to create Volume with name: %s, err: %v", spec.K8sVolumeName, err))
+		return nil,
+			status.Errorf(codes.Internal, fmt.Sprintf("Failed to create LUN, err: %v", err))
 	}
 
 	// No matter lun existed or not, Get Lun by name
 	lunInfo, err := dsm.LunGet(spec.LunName)
 	if err != nil {
-		return webapi.LunInfo{},
+		return nil,
 			// discussion with log
 			status.Errorf(codes.Internal, fmt.Sprintf("Failed to get existed LUN with name: %s, err: %v", spec.LunName, err))
 	}
 
 	// 4. Create Target and Map to Lun
-	err = service.createMappingTarget(dsm, spec, lunInfo.Uuid)
+	targetInfo, err := service.createMappingTarget(dsm, spec, lunInfo.Uuid)
 	if err != nil {
 		// FIXME need to delete lun and target
-		return webapi.LunInfo{},
+		return nil,
 			status.Errorf(codes.Internal, fmt.Sprintf("Failed to create and map target, err: %v", err))
 	}
 
-	log.Debugf("[%s] CreateVolume Successfully. VolumeId: %s", dsm.Ip, lunInfo.Uuid);
+	log.Debugf("[%s] CreateVolume Successfully. VolumeId: %s", dsm.Ip, lunInfo.Uuid)
 
-	return lunInfo, nil
+	return DsmLunToK8sVolume(dsm.Ip, lunInfo, targetInfo), nil
 }
 
 func waitCloneFinished(dsm *webapi.DSM, lunName string) error {
@@ -291,47 +291,47 @@ func waitCloneFinished(dsm *webapi.DSM, lunName string) error {
 	return nil
 }
 
-func (service *DsmService) createVolumeBySnapshot(dsm *webapi.DSM, spec *models.CreateK8sVolumeSpec, snapshotInfo webapi.SnapshotInfo) (webapi.LunInfo, error) {
-	if spec.Size != 0 && spec.Size != snapshotInfo.TotalSize {
-		return webapi.LunInfo{}, status.Errorf(codes.OutOfRange, "Lun size [%d] is not equal to snapshot size [%d]", spec.Size, snapshotInfo.TotalSize)
+func (service *DsmService) createVolumeBySnapshot(dsm *webapi.DSM, spec *models.CreateK8sVolumeSpec, srcSnapshot *models.K8sSnapshotRespSpec) (*models.K8sVolumeRespSpec, error) {
+	if spec.Size != 0 && spec.Size != srcSnapshot.SizeInBytes {
+		return nil, status.Errorf(codes.OutOfRange, "Requested lun size [%d] is not equal to snapshot size [%d]", spec.Size, srcSnapshot.SizeInBytes)
 	}
 
 	snapshotCloneSpec := webapi.SnapshotCloneSpec{
 		Name:            spec.LunName,
-		SrcLunUuid:      snapshotInfo.ParentUuid,
-		SrcSnapshotUuid: snapshotInfo.Uuid,
+		SrcLunUuid:      srcSnapshot.ParentUuid,
+		SrcSnapshotUuid: srcSnapshot.Uuid,
 	}
 
 	if _, err := dsm.SnapshotClone(snapshotCloneSpec); err != nil && !errors.Is(err, utils.AlreadyExistError("")) {
-		return webapi.LunInfo{},
-			status.Errorf(codes.Internal, fmt.Sprintf("Failed to create volume with source snapshot ID: %s, err: %v", snapshotInfo.Uuid, err))
+		return nil,
+			status.Errorf(codes.Internal, fmt.Sprintf("Failed to create volume with source snapshot ID: %s, err: %v", srcSnapshot.Uuid, err))
 	}
 
 	if err := waitCloneFinished(dsm, spec.LunName); err != nil {
-		return webapi.LunInfo{}, status.Errorf(codes.Internal, err.Error())
+		return nil, status.Errorf(codes.Internal, err.Error())
 	}
 
 	lunInfo, err := dsm.LunGet(spec.LunName)
 	if err != nil {
-		return webapi.LunInfo{},
+		return nil,
 			status.Errorf(codes.Internal, fmt.Sprintf("Failed to get existed LUN with name: %s, err: %v", spec.LunName, err))
 	}
 
-	err = service.createMappingTarget(dsm, spec, lunInfo.Uuid)
+	targetInfo, err := service.createMappingTarget(dsm, spec, lunInfo.Uuid)
 	if err != nil {
 		// FIXME need to delete lun and target
-		return webapi.LunInfo{},
+		return nil,
 			status.Errorf(codes.Internal, fmt.Sprintf("Failed to create and map target, err: %v", err))
 	}
 
-	log.Debugf("[%s] createVolumeBySnapshot Successfully. VolumeId: %s", dsm.Ip, lunInfo.Uuid);
+	log.Debugf("[%s] createVolumeBySnapshot Successfully. VolumeId: %s", dsm.Ip, lunInfo.Uuid)
 
-	return lunInfo, nil
+	return DsmLunToK8sVolume(dsm.Ip, lunInfo, targetInfo), nil
 }
 
-func (service *DsmService) createVolumeByVolume(dsm *webapi.DSM, spec *models.CreateK8sVolumeSpec, srcLunInfo webapi.LunInfo) (webapi.LunInfo, error) {
+func (service *DsmService) createVolumeByVolume(dsm *webapi.DSM, spec *models.CreateK8sVolumeSpec, srcLunInfo webapi.LunInfo) (*models.K8sVolumeRespSpec, error) {
 	if spec.Size != 0 && spec.Size != int64(srcLunInfo.Size) {
-		return webapi.LunInfo{}, status.Errorf(codes.OutOfRange, "Lun size [%d] is not equal to src lun size [%d]", spec.Size, srcLunInfo.Size)
+		return nil, status.Errorf(codes.OutOfRange, "Requested lun size [%d] is not equal to src lun size [%d]", spec.Size, srcLunInfo.Size)
 	}
 
 	if spec.Location == "" {
@@ -345,154 +345,230 @@ func (service *DsmService) createVolumeByVolume(dsm *webapi.DSM, spec *models.Cr
 	}
 
 	if _, err := dsm.LunClone(lunCloneSpec); err != nil && !errors.Is(err, utils.AlreadyExistError("")) {
-		return webapi.LunInfo{},
+		return nil,
 			status.Errorf(codes.Internal, fmt.Sprintf("Failed to create volume with source volume ID: %s, err: %v", srcLunInfo.Uuid, err))
 	}
 
 	if err := waitCloneFinished(dsm, spec.LunName); err != nil {
-		return webapi.LunInfo{}, status.Errorf(codes.Internal, err.Error())
+		return nil, status.Errorf(codes.Internal, err.Error())
 	}
 
 	lunInfo, err := dsm.LunGet(spec.LunName)
 	if err != nil {
-		return webapi.LunInfo{},
+		return nil,
 			status.Errorf(codes.Internal, fmt.Sprintf("Failed to get existed LUN with name: %s, err: %v", spec.LunName, err))
 	}
 
-	err = service.createMappingTarget(dsm, spec, lunInfo.Uuid)
+	targetInfo, err := service.createMappingTarget(dsm, spec, lunInfo.Uuid)
 	if err != nil {
 		// FIXME need to delete lun and target
-		return webapi.LunInfo{},
+		return nil,
 			status.Errorf(codes.Internal, fmt.Sprintf("Failed to create and map target, err: %v", err))
 	}
 
-	log.Debugf("[%s] createVolumeByVolume Successfully. VolumeId: %s", dsm.Ip, lunInfo.Uuid);
+	log.Debugf("[%s] createVolumeByVolume Successfully. VolumeId: %s", dsm.Ip, lunInfo.Uuid)
 
-	return lunInfo, nil
+	return DsmLunToK8sVolume(dsm.Ip, lunInfo, targetInfo), nil
 }
 
-func (service *DsmService) CreateVolume(spec *models.CreateK8sVolumeSpec) (webapi.LunInfo, string, error) {
+func DsmShareToK8sVolume(dsmIp string, info webapi.ShareInfo) *models.K8sVolumeRespSpec {
+	return &models.K8sVolumeRespSpec{
+		DsmIp: dsmIp,
+		VolumeId: info.Uuid,
+		SizeInBytes: utils.MBToBytes(info.QuotaValueInMB),
+		Location: info.VolPath,
+		Name: info.Name,
+		Source: "//" + dsmIp + "/" + info.Name,
+		Protocol: utils.ProtocolSmb,
+		Share: info,
+	}
+}
+
+func DsmLunToK8sVolume(dsmIp string, info webapi.LunInfo, targetInfo webapi.TargetInfo) *models.K8sVolumeRespSpec {
+	return &models.K8sVolumeRespSpec{
+		DsmIp: dsmIp,
+		VolumeId: info.Uuid,
+		SizeInBytes: int64(info.Size),
+		Location: info.Location,
+		Name: info.Name,
+		Source: "",
+		Protocol: utils.ProtocolIscsi,
+		Lun: info,
+		Target: targetInfo,
+	}
+}
+
+func (service *DsmService) CreateVolume(spec *models.CreateK8sVolumeSpec) (*models.K8sVolumeRespSpec, error) {
 	if spec.SourceVolumeId != "" {
 		/* Create volume by exists volume (Clone) */
 		k8sVolume := service.GetVolume(spec.SourceVolumeId)
 		if k8sVolume == nil {
-			return webapi.LunInfo{}, "", status.Errorf(codes.NotFound, fmt.Sprintf("No such volume id: %s", spec.SourceVolumeId))
+			return nil, status.Errorf(codes.NotFound, fmt.Sprintf("No such volume id: %s", spec.SourceVolumeId))
 		}
 
 		dsm, err := service.GetDsm(k8sVolume.DsmIp)
 		if err != nil {
-			return webapi.LunInfo{}, "", status.Errorf(codes.Internal, fmt.Sprintf("Failed to get DSM[%s]", k8sVolume.DsmIp))
+			return nil, status.Errorf(codes.Internal, fmt.Sprintf("Failed to get DSM[%s]", k8sVolume.DsmIp))
 		}
 
-		lunInfo, err := service.createVolumeByVolume(dsm, spec, k8sVolume.Lun)
-		return lunInfo, dsm.Ip, err
-	} else if spec.SourceSnapshotId != "" {
-		/* Create volume by snapshot */
-		for _, dsm := range service.dsms {
-			snapshotInfo, err := dsm.SnapshotGet(spec.SourceSnapshotId)
-			if err != nil {
-				continue
-			}
-
-			lunInfo, err := service.createVolumeBySnapshot(dsm, spec, snapshotInfo)
-			return lunInfo, dsm.Ip, err
+		if spec.Protocol == utils.ProtocolIscsi {
+			return service.createVolumeByVolume(dsm, spec, k8sVolume.Lun)
+		} else if spec.Protocol == utils.ProtocolSmb {
+			return service.createSMBVolumeByVolume(dsm, spec, k8sVolume.Share)
 		}
-		return webapi.LunInfo{}, "", status.Errorf(codes.NotFound, fmt.Sprintf("No such snapshot id: %s", spec.SourceSnapshotId))
-	} else if spec.DsmIp != "" {
-		/* Create volume by specific dsm ip */
-		dsm, err := service.GetDsm(spec.DsmIp)
-		if err != nil {
-			return webapi.LunInfo{}, "", status.Errorf(codes.Internal, fmt.Sprintf("%v", err))
-		}
-		lunInfo, err := service.createVolumeByDsm(dsm, spec)
-		return lunInfo, dsm.Ip, err
-	} else {
-		/* Find appropriate dsm to create volume */
-		for _, dsm := range service.dsms {
-			lunInfo, err := service.createVolumeByDsm(dsm, spec)
-			if err != nil {
-				log.Errorf("[%s] Failed to create Volume: %v", dsm.Ip, err)
-				continue
-			}
-			return lunInfo, dsm.Ip, nil
-		}
-		return webapi.LunInfo{}, "", status.Errorf(codes.Internal, fmt.Sprintf("Couldn't find any host available to create Volume"))
+		return nil, status.Error(codes.InvalidArgument, "Unknown protocol")
 	}
+
+	if spec.SourceSnapshotId != "" {
+		/* Create volume by snapshot */
+		snapshot := service.GetSnapshotByUuid(spec.SourceSnapshotId)
+		if snapshot == nil {
+			return nil, status.Errorf(codes.NotFound, fmt.Sprintf("No such snapshot id: %s", spec.SourceSnapshotId))
+		}
+
+		// found source by snapshot id, check allowable
+		if spec.DsmIp != "" && spec.DsmIp != snapshot.DsmIp {
+			msg := fmt.Sprintf("The source PVC and destination PVCs must be on the same DSM for cloning from snapshots. Source is on %s, but new PVC is on %s",
+				snapshot.DsmIp, spec.DsmIp)
+			return nil, status.Errorf(codes.InvalidArgument, msg)
+		}
+		if spec.Location != "" && spec.Location != snapshot.RootPath {
+			msg := fmt.Sprintf("The source PVC and destination PVCs must be on the same location for cloning from snapshots. Source is on %s, but new PVC is on %s",
+				snapshot.RootPath, spec.Location)
+			return nil, status.Errorf(codes.InvalidArgument, msg)
+		}
+		if spec.Protocol != snapshot.Protocol {
+			msg := fmt.Sprintf("The source PVC and destination PVCs shouldn't have different protocols. Source is %s, but new PVC is %s",
+				snapshot.Protocol, spec.Protocol)
+			return nil, status.Errorf(codes.InvalidArgument, msg)
+		}
+
+		dsm, err := service.GetDsm(snapshot.DsmIp)
+		if err != nil {
+			return nil, status.Errorf(codes.Internal, fmt.Sprintf("Failed to get DSM[%s]", snapshot.DsmIp))
+		}
+
+		if spec.Protocol == utils.ProtocolIscsi {
+			return service.createVolumeBySnapshot(dsm, spec, snapshot)
+		} else if spec.Protocol == utils.ProtocolSmb {
+			return service.createSMBVolumeBySnapshot(dsm, spec, snapshot)
+		}
+		return nil, status.Error(codes.InvalidArgument, "Unknown protocol")
+	}
+
+	/* Find appropriate dsm to create volume */
+	for _, dsm := range service.dsms {
+		if spec.DsmIp != "" && spec.DsmIp != dsm.Ip {
+			continue
+		}
+
+		var k8sVolume *models.K8sVolumeRespSpec
+		var err error
+		if spec.Protocol == utils.ProtocolIscsi {
+			k8sVolume, err = service.createVolumeByDsm(dsm, spec)
+		} else if spec.Protocol == utils.ProtocolSmb {
+			k8sVolume, err = service.createSMBVolumeByDsm(dsm, spec)
+		}
+
+		if err != nil {
+			log.Errorf("[%s] Failed to create Volume: %v", dsm.Ip, err)
+			continue
+		}
+
+		return k8sVolume, nil
+	}
+
+	return nil, status.Errorf(codes.Internal, fmt.Sprintf("Couldn't find any host available to create Volume"))
 }
 
-func (service *DsmService) DeleteVolume(volumeId string) error {
-	k8sVolume := service.GetVolume(volumeId)
+func (service *DsmService) DeleteVolume(volId string) error {
+	k8sVolume := service.GetVolume(volId)
 	if k8sVolume == nil {
-		log.Infof("Skip delete volume[%s] that is no exist", volumeId)
+		log.Infof("Skip delete volume[%s] that is no exist", volId)
 		return nil
 	}
 
-	lun, target := k8sVolume.Lun, k8sVolume.Target
 	dsm, err := service.GetDsm(k8sVolume.DsmIp)
-
 	if err != nil {
 		return status.Errorf(codes.Internal, fmt.Sprintf("Failed to get DSM[%s]", k8sVolume.DsmIp))
 	}
 
-	if err := dsm.LunDelete(lun.Uuid); err != nil {
-		return err
-	}
+	if k8sVolume.Protocol == utils.ProtocolSmb {
+		if err := dsm.ShareDelete(k8sVolume.Share.Name); err != nil {
+			log.Errorf("[%s] Failed to delete Share(%s): %v", dsm.Ip, k8sVolume.Share.Name, err)
+			return err
+		}
+	} else {
+		lun, target := k8sVolume.Lun, k8sVolume.Target
 
-	if len(target.MappedLuns) != 1 {
-		log.Infof("Skip deletes target[%s] that was mapped with lun. DSM[%s]", target.Name, dsm.Ip)
-		return nil
-	}
+		if err := dsm.LunDelete(lun.Uuid); err != nil {
+			if  _, err := dsm.LunGet(lun.Uuid); err != nil && errors.Is(err, utils.NoSuchLunError("")) {
+				return nil
+			}
+			log.Errorf("[%s] Failed to delete LUN(%s): %v", dsm.Ip, lun.Uuid, err)
+			return err
+		}
 
-	if err := dsm.TargetDelete(strconv.Itoa(target.TargetId)); err != nil {
-		return err
+		if len(target.MappedLuns) != 1 {
+			log.Infof("Skip deletes target[%s] that was mapped with lun. DSM[%s]", target.Name, dsm.Ip)
+			return nil
+		}
+
+		if err := dsm.TargetDelete(strconv.Itoa(target.TargetId)); err != nil {
+			if  _, err := dsm.TargetGet(strconv.Itoa(target.TargetId)); err != nil {
+				return nil
+			}
+			log.Errorf("[%s] Failed to delete target(%d): %v", dsm.Ip, target.TargetId, err)
+			return err
+		}
 	}
 
 	return nil
 }
 
-func (service *DsmService) listVolumesByDsm(dsm *webapi.DSM, infos *[]*models.ListK8sVolumeRespSpec) {
-	targetInfos, err := dsm.TargetList()
-	if err != nil {
-		log.Errorf("[%s] Failed to list targets: %v", dsm.Ip, err)
-	}
+func (service *DsmService) listISCSIVolumes(dsmIp string) (infos []*models.K8sVolumeRespSpec) {
+	for _, dsm := range service.dsms {
+		if dsmIp != "" && dsmIp != dsm.Ip {
+			continue
+		}
 
-	for _, target := range targetInfos {
-		// TODO: use target.ConnectedSessions to filter targets
-		for _, mapping := range target.MappedLuns {
-			lun, err := dsm.LunGet(mapping.LunUuid)
-			if err != nil {
-				log.Errorf("[%s] Failed to get LUN(%s): %v", dsm.Ip, mapping.LunUuid, err)
+		targetInfos, err := dsm.TargetList()
+		if err != nil {
+			log.Errorf("[%s] Failed to list targets: %v", dsm.Ip, err)
+			continue
+		}
+
+		for _, target := range targetInfos {
+			// TODO: use target.ConnectedSessions to filter targets
+			for _, mapping := range target.MappedLuns {
+				lun, err := dsm.LunGet(mapping.LunUuid)
+				if err != nil {
+					log.Errorf("[%s] Failed to get LUN(%s): %v", dsm.Ip, mapping.LunUuid, err)
+				}
+
+				if !strings.HasPrefix(lun.Name, models.LunPrefix) {
+					continue
+				}
+
+				// FIXME: filter same LUN mapping to two target
+				infos = append(infos, DsmLunToK8sVolume(dsm.Ip, lun, target))
 			}
-
-			if !strings.HasPrefix(lun.Name, models.LunPrefix) {
-				continue
-			}
-
-			*infos = append(*infos, &models.ListK8sVolumeRespSpec{
-				DsmIp:  dsm.Ip,
-				Target: target,
-				Lun:    lun,
-			})
 		}
 	}
-	return
+	return infos
 }
 
-func (service *DsmService) ListVolumes() []*models.ListK8sVolumeRespSpec {
-	var infos []*models.ListK8sVolumeRespSpec
-
-	for _, dsm := range service.dsms {
-		service.listVolumesByDsm(dsm, &infos)
-	}
+func (service *DsmService) ListVolumes() (infos []*models.K8sVolumeRespSpec) {
+	infos = append(infos, service.listISCSIVolumes("")...)
+	infos = append(infos, service.listSMBVolumes("")...)
 
 	return infos
 }
 
-func (service *DsmService) GetVolume(lunUuid string) *models.ListK8sVolumeRespSpec {
+func (service *DsmService) GetVolume(volId string) *models.K8sVolumeRespSpec {
 	volumes := service.ListVolumes()
-
 	for _, volume := range volumes {
-		if volume.Lun.Uuid == lunUuid {
+		if volume.VolumeId == volId {
 			return volume
 		}
 	}
@@ -500,100 +576,74 @@ func (service *DsmService) GetVolume(lunUuid string) *models.ListK8sVolumeRespSp
 	return nil
 }
 
-func (service *DsmService) ExpandLun(lunUuid string, newSize int64) error {
-	k8sVolume := service.GetVolume(lunUuid);
-	if k8sVolume == nil {
-		return status.Errorf(codes.InvalidArgument, fmt.Sprintf("Can't find volume[%s].", lunUuid))
-	}
-
-	if int64(k8sVolume.Lun.Size) > newSize {
-		return status.Errorf(codes.InvalidArgument,
-			fmt.Sprintf("Failed to expand volume[%s], because expand size[%d] bigger than before[%d].",
-				lunUuid, newSize, k8sVolume.Lun.Size))
-	}
-
-	spec := webapi.LunUpdateSpec{
-		Uuid: lunUuid,
-		NewSize: uint64(newSize),
-	}
-
-	dsm, err := service.GetDsm(k8sVolume.DsmIp)
-	if err != nil {
-		return status.Errorf(codes.Internal, fmt.Sprintf("Failed to get DSM[%s]", k8sVolume.DsmIp))
-	}
-
-	if err := dsm.LunUpdate(spec); err != nil {
-		return status.Errorf(codes.InvalidArgument,
-			fmt.Sprintf("Failed to expand volume[%s]. err: %v", lunUuid, err))
+func (service *DsmService) GetVolumeByName(volName string) *models.K8sVolumeRespSpec {
+	volumes := service.ListVolumes()
+	for _, volume := range volumes {
+		if volume.Name == models.GenLunName(volName) ||
+			volume.Name == models.GenShareName(volName) {
+			return volume
+		}
 	}
 
 	return nil
 }
 
-func (service *DsmService) CreateSnapshot(spec *models.CreateK8sVolumeSnapshotSpec) (string, error) {
-	k8sVolume := service.GetVolume(spec.K8sVolumeId);
+func (service *DsmService) GetSnapshotByName(snapshotName string) *models.K8sSnapshotRespSpec {
+	snaps := service.ListAllSnapshots()
+	for _, snap := range snaps {
+		if snap.Name == snapshotName {
+			return snap
+		}
+	}
+	return nil
+}
+
+func (service *DsmService) ExpandVolume(volId string, newSize int64) (*models.K8sVolumeRespSpec, error) {
+	k8sVolume := service.GetVolume(volId);
 	if k8sVolume == nil {
-		return "", status.Errorf(codes.NotFound, fmt.Sprintf("Can't find volume[%s].", spec.K8sVolumeId))
+		return nil, status.Errorf(codes.InvalidArgument, fmt.Sprintf("Can't find volume[%s].", volId))
 	}
 
-	snapshotSpec := webapi.SnapshotCreateSpec{
-		Name:    spec.SnapshotName,
-		LunUuid: spec.K8sVolumeId,
-		Description: spec.Description,
-		TakenBy: spec.TakenBy,
-		IsLocked: spec.IsLocked,
+	if k8sVolume.SizeInBytes > newSize {
+		return nil, status.Errorf(codes.InvalidArgument,
+			fmt.Sprintf("Failed to expand volume[%s], because expand size[%d] smaller than before[%d].",
+				volId, newSize, k8sVolume.SizeInBytes))
 	}
 
 	dsm, err := service.GetDsm(k8sVolume.DsmIp)
 	if err != nil {
-		return "", status.Errorf(codes.InvalidArgument, fmt.Sprintf("Failed to get dsm: %v", err))
+		return nil, status.Errorf(codes.Internal, fmt.Sprintf("Failed to get DSM[%s]", k8sVolume.DsmIp))
 	}
 
-	snapshotUuid, err := dsm.SnapshotCreate(snapshotSpec)
-	if err != nil {
-		return "", err
+	if k8sVolume.Protocol == utils.ProtocolSmb {
+		newSizeInMB := utils.BytesToMBCeil(newSize) // round up to MB
+		if err := dsm.SetShareQuota(k8sVolume.Share, newSizeInMB); err != nil {
+			log.Errorf("[%s] Failed to set quota [%d (MB)] to Share [%s]: %v",
+				dsm.Ip, newSizeInMB, k8sVolume.Share.Name, err)
+			return nil, status.Errorf(codes.Internal, fmt.Sprintf("Failed to expand volume[%s]. err: %v", volId, err))
+		}
+		// convert MB to bytes, may be diff from the input newSize
+		k8sVolume.SizeInBytes = utils.MBToBytes(newSizeInMB)
+	} else {
+		spec := webapi.LunUpdateSpec{
+			Uuid: volId,
+			NewSize: uint64(newSize),
+		}
+		if err := dsm.LunUpdate(spec); err != nil {
+			return nil, status.Errorf(codes.Internal, fmt.Sprintf("Failed to expand volume[%s]. err: %v", volId, err))
+		}
+		k8sVolume.SizeInBytes = newSize
 	}
 
-	return snapshotUuid, nil
+	return k8sVolume, nil
 }
 
-func (service *DsmService) DeleteSnapshot(snapshotUuid string) error {
-	for _, dsm := range service.dsms {
-		_, err := dsm.SnapshotGet(snapshotUuid)
-		if err != nil {
-			continue
-		}
+func (service *DsmService) CreateSnapshot(spec *models.CreateK8sVolumeSnapshotSpec) (*models.K8sSnapshotRespSpec, error) {
+	srcVolId := spec.K8sVolumeId
 
-		err = dsm.SnapshotDelete(snapshotUuid)
-		if err != nil {
-			return status.Errorf(codes.Internal, fmt.Sprintf("Failed to delete snapshot [%s]. err: %v", snapshotUuid, err))
-		}
-
-		return nil
-	}
-
-	return nil
-}
-
-func (service *DsmService) ListAllSnapshots() ([]webapi.SnapshotInfo, error) {
-	var allInfos []webapi.SnapshotInfo
-
-	for _, dsm := range service.dsms {
-		infos, err := dsm.SnapshotList("")
-		if err != nil {
-			continue
-		}
-
-		allInfos = append(allInfos, infos...)
-	}
-
-	return allInfos, nil
-}
-
-func (service *DsmService) ListSnapshots(lunUuid string) ([]webapi.SnapshotInfo, error) {
-	k8sVolume := service.GetVolume(lunUuid);
+	k8sVolume := service.GetVolume(srcVolId);
 	if k8sVolume == nil {
-		return []webapi.SnapshotInfo{}, nil // return empty when the volume does not exist
+		return nil, status.Errorf(codes.NotFound, fmt.Sprintf("Can't find volume[%s].", srcVolId))
 	}
 
 	dsm, err := service.GetDsm(k8sVolume.DsmIp)
@@ -601,24 +651,201 @@ func (service *DsmService) ListSnapshots(lunUuid string) ([]webapi.SnapshotInfo,
 		return nil, status.Errorf(codes.InvalidArgument, fmt.Sprintf("Failed to get dsm: %v", err))
 	}
 
-	infos, err := dsm.SnapshotList(lunUuid)
-	if err != nil {
-		return nil, status.Errorf(codes.Internal,
-			fmt.Sprintf("Failed to list snapshot on lun [%s]. err: %v", lunUuid, err))
+	if k8sVolume.Protocol == utils.ProtocolIscsi {
+		snapshotSpec := webapi.SnapshotCreateSpec{
+			Name:    spec.SnapshotName,
+			LunUuid: srcVolId,
+			Description: spec.Description,
+			TakenBy: spec.TakenBy,
+			IsLocked: spec.IsLocked,
+		}
+
+		snapshotUuid, err := dsm.SnapshotCreate(snapshotSpec)
+		if err != nil {
+			if err == utils.OutOfFreeSpaceError("") || err == utils.SnapshotReachMaxCountError("") {
+				return nil,status.Errorf(codes.ResourceExhausted, fmt.Sprintf("Failed to SnapshotCreate(%s), err: %v", srcVolId, err))
+			}
+			return nil, status.Errorf(codes.Internal, fmt.Sprintf("Failed to SnapshotCreate(%s), err: %v", srcVolId, err))
+		}
+
+		if snapshot := service.getISCSISnapshot(snapshotUuid); snapshot != nil {
+			return snapshot, nil
+		}
+
+		return nil, status.Errorf(codes.NotFound, fmt.Sprintf("Failed to get iscsi snapshot (%s). Not found", snapshotUuid))
+	} else if k8sVolume.Protocol == utils.ProtocolSmb {
+		snapshotSpec := webapi.ShareSnapshotCreateSpec{
+			ShareName: k8sVolume.Share.Name,
+			Desc:      models.ShareSnapshotDescPrefix + spec.SnapshotName, // limitations: don't change the desc by DSM
+			IsLocked:  spec.IsLocked,
+		}
+
+		snapshotTime, err := dsm.ShareSnapshotCreate(snapshotSpec)
+		if err != nil {
+			return nil, status.Errorf(codes.Internal, fmt.Sprintf("Failed to ShareSnapshotCreate(%s), err: %v", srcVolId, err))
+		}
+
+		snapshots := service.listSMBSnapshotsByDsm(dsm)
+		for _, snapshot := range snapshots {
+			if snapshot.Time == snapshotTime && snapshot.ParentUuid == srcVolId {
+				return snapshot, nil
+			}
+		}
+		return nil, status.Errorf(codes.NotFound, fmt.Sprintf("Failed to get smb snapshot (%s, %s). Not found", snapshotTime, srcVolId))
 	}
 
-	return infos, nil
+	return nil, status.Error(codes.InvalidArgument, "Unsupported volume protocol")
 }
 
-func (service *DsmService) GetSnapshot(snapshotUuid string) (webapi.SnapshotInfo, error) {
-	for _, dsm := range service.dsms {
-		info, err := dsm.SnapshotGet(snapshotUuid)
+func (service *DsmService) GetSnapshotByUuid(snapshotUuid string) *models.K8sSnapshotRespSpec {
+	snaps := service.ListAllSnapshots()
+	for _, snap := range snaps {
+		if snap.Uuid == snapshotUuid {
+			return snap
+		}
+	}
+	return nil
+}
+
+func (service *DsmService) DeleteSnapshot(snapshotUuid string) error {
+	snapshot := service.GetSnapshotByUuid(snapshotUuid)
+	if snapshot == nil {
+		return nil
+	}
+	dsm, err := service.GetDsm(snapshot.DsmIp)
+	if err != nil {
+		return err
+	}
+
+	if snapshot.Protocol == utils.ProtocolSmb {
+		if err := dsm.ShareSnapshotDelete(snapshot.Time, snapshot.ParentName); err != nil {
+			if snapshot := service.getSMBSnapshot(snapshotUuid); snapshot == nil { // idempotency
+				return nil
+			}
+
+			log.Errorf("Failed to delete Share snapshot [%s]. err: %v", snapshotUuid, err)
+			return err
+		}
+	} else if snapshot.Protocol == utils.ProtocolIscsi {
+		if err := dsm.SnapshotDelete(snapshotUuid); err != nil {
+			if _, err := dsm.SnapshotGet(snapshotUuid); err != nil { // idempotency
+				return nil
+			}
+
+			log.Errorf("Failed to delete LUN snapshot [%s]. err: %v", snapshotUuid, err)
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (service *DsmService) listISCSISnapshotsByDsm(dsm *webapi.DSM) (infos []*models.K8sSnapshotRespSpec) {
+	volumes := service.listISCSIVolumes(dsm.Ip)
+	for _, volume := range volumes {
+		lunInfo := volume.Lun
+		lunSnaps, err := dsm.SnapshotList(lunInfo.Uuid)
 		if err != nil {
+			log.Errorf("[%s] Failed to list LUN[%s] snapshots: %v", dsm.Ip, lunInfo.Uuid, err)
 			continue
 		}
 
-		return info, nil
+		for _, info := range lunSnaps {
+			infos = append(infos, DsmLunSnapshotToK8sSnapshot(dsm.Ip, info, lunInfo))
+		}
+	}
+	return
+}
+
+func (service *DsmService) ListAllSnapshots() []*models.K8sSnapshotRespSpec {
+	var allInfos []*models.K8sSnapshotRespSpec
+
+	for _, dsm := range service.dsms {
+		allInfos = append(allInfos, service.listISCSISnapshotsByDsm(dsm)...)
+		allInfos = append(allInfos, service.listSMBSnapshotsByDsm(dsm)...)
 	}
 
-	return webapi.SnapshotInfo{}, status.Errorf(codes.Internal, fmt.Sprintf("No such snapshot uuid [%s]", snapshotUuid))
+	return allInfos
+}
+
+func (service *DsmService) ListSnapshots(volId string) []*models.K8sSnapshotRespSpec {
+	var allInfos []*models.K8sSnapshotRespSpec
+
+	k8sVolume := service.GetVolume(volId);
+	if k8sVolume == nil {
+		return nil
+	}
+
+	dsm, err := service.GetDsm(k8sVolume.DsmIp)
+	if err != nil {
+		log.Errorf("Failed to get DSM[%s]", k8sVolume.DsmIp)
+		return nil
+	}
+
+	if k8sVolume.Protocol == utils.ProtocolIscsi {
+		infos, err := dsm.SnapshotList(volId)
+		if err != nil {
+			log.Errorf("Failed to SnapshotList[%s]", volId)
+			return nil
+		}
+		for _, info := range infos {
+			allInfos = append(allInfos, DsmLunSnapshotToK8sSnapshot(dsm.Ip, info, k8sVolume.Lun))
+		}
+	} else {
+		infos, err := dsm.ShareSnapshotList(k8sVolume.Share.Name)
+		if err != nil {
+			log.Errorf("Failed to ShareSnapshotList[%s]", k8sVolume.Share.Name)
+			return nil
+		}
+		for _, info := range infos {
+			allInfos = append(allInfos, DsmShareSnapshotToK8sSnapshot(dsm.Ip, info, k8sVolume.Share))
+		}
+	}
+
+	return allInfos
+}
+
+func DsmShareSnapshotToK8sSnapshot(dsmIp string, info webapi.ShareSnapshotInfo, shareInfo webapi.ShareInfo) *models.K8sSnapshotRespSpec {
+	return &models.K8sSnapshotRespSpec{
+		DsmIp: dsmIp,
+		Name: strings.ReplaceAll(info.Desc, models.ShareSnapshotDescPrefix, ""), // snapshot-XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX
+		Uuid: info.Uuid,
+		ParentName: shareInfo.Name,
+		ParentUuid: shareInfo.Uuid,
+		Status: "Healthy", // share snapshot always Healthy
+		SizeInBytes: utils.MBToBytes(shareInfo.QuotaValueInMB), // unable to get snapshot quota, return parent quota instead
+		CreateTime: GMTToUnixSecond(info.Time),
+		Time: info.Time,
+		RootPath: shareInfo.VolPath,
+		Protocol: utils.ProtocolSmb,
+	}
+}
+
+func DsmLunSnapshotToK8sSnapshot(dsmIp string, info webapi.SnapshotInfo, lunInfo webapi.LunInfo) *models.K8sSnapshotRespSpec {
+	return &models.K8sSnapshotRespSpec{
+		DsmIp: dsmIp,
+		Name: info.Name,
+		Uuid: info.Uuid,
+		ParentName: lunInfo.Name, // it can be empty for iscsi
+		ParentUuid: info.ParentUuid,
+		Status: info.Status,
+		SizeInBytes: info.TotalSize,
+		CreateTime: info.CreateTime,
+		Time: "",
+		RootPath: info.RootPath,
+		Protocol: utils.ProtocolIscsi,
+	}
+}
+
+func (service *DsmService) getISCSISnapshot(snapshotUuid string) *models.K8sSnapshotRespSpec {
+	for _, dsm := range service.dsms {
+		snapshots := service.listISCSISnapshotsByDsm(dsm)
+		for _, snap := range snapshots {
+			if snap.Uuid == snapshotUuid {
+				return snap
+			}
+		}
+	}
+
+	return nil
 }

pkg/dsm/service/share_volume.go

@@ -0,0 +1,231 @@
+/*
+ * Copyright 2022 Synology Inc.
+ */
+
+package service
+
+import (
+	"errors"
+	"fmt"
+	log "github.com/sirupsen/logrus"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
+	"strings"
+	"time"
+
+	"github.com/SynologyOpenSource/synology-csi/pkg/dsm/webapi"
+	"github.com/SynologyOpenSource/synology-csi/pkg/models"
+	"github.com/SynologyOpenSource/synology-csi/pkg/utils"
+)
+
+func GMTToUnixSecond(timeStr string) (int64) {
+	t, err := time.Parse("GMT-07-2006.01.02-15.04.05", timeStr)
+	if err != nil {
+		log.Error(err)
+		return -1
+	}
+	return t.Unix()
+}
+
+func (service *DsmService) createSMBVolumeBySnapshot(dsm *webapi.DSM, spec *models.CreateK8sVolumeSpec, srcSnapshot *models.K8sSnapshotRespSpec) (*models.K8sVolumeRespSpec, error) {
+	srcShareInfo, err := dsm.ShareGet(srcSnapshot.ParentName)
+	if err != nil {
+		return nil, status.Errorf(codes.Internal, fmt.Sprintf("Failed to get share: %s, err: %v", srcSnapshot.ParentName, err))
+	}
+
+	shareCloneSpec := webapi.ShareCloneSpec{
+		Name: spec.ShareName,
+		Snapshot: srcSnapshot.Time,
+		ShareInfo: webapi.ShareInfo{
+			Name:                spec.ShareName,
+			VolPath:             srcSnapshot.RootPath,
+			Desc:                "Cloned from [" + srcSnapshot.Time + "] by csi driver", // max: 64
+			EnableRecycleBin:    srcShareInfo.EnableRecycleBin,
+			RecycleBinAdminOnly: srcShareInfo.RecycleBinAdminOnly,
+			NameOrg:             srcSnapshot.ParentName,
+		},
+	}
+
+	if _, err := dsm.ShareClone(shareCloneSpec); err != nil && !errors.Is(err, utils.AlreadyExistError("")) {
+		return nil,
+			status.Errorf(codes.Internal, fmt.Sprintf("Failed to create volume with source volume ID: %s, err: %v", srcShareInfo.Uuid, err))
+	}
+
+	shareInfo, err := dsm.ShareGet(spec.ShareName)
+	if err != nil {
+		return nil,
+			status.Errorf(codes.Internal, fmt.Sprintf("Failed to get existed Share with name: [%s], err: %v", spec.ShareName, err))
+	}
+
+	newSizeInMB := utils.BytesToMBCeil(spec.Size)
+	if shareInfo.QuotaValueInMB == 0 {
+		// known issue for some DS, manually set quota to the new share
+		if err := dsm.SetShareQuota(shareInfo, newSizeInMB); err != nil {
+			msg := fmt.Sprintf("Failed to set quota [%d] to Share [%s], err: %v", newSizeInMB, shareInfo.Name, err)
+			log.Error(msg)
+			return nil, status.Errorf(codes.Internal, msg)
+		}
+
+		shareInfo.QuotaValueInMB = newSizeInMB
+	}
+
+	if newSizeInMB != shareInfo.QuotaValueInMB {
+		// FIXME: need to delete share
+		return nil,
+			status.Errorf(codes.OutOfRange, "Requested share quotaMB [%d] is not equal to snapshot restore quotaMB [%d]", newSizeInMB, shareInfo.QuotaValueInMB)
+	}
+
+	log.Debugf("[%s] createSMBVolumeBySnapshot Successfully. VolumeId: %s", dsm.Ip, shareInfo.Uuid);
+
+	return DsmShareToK8sVolume(dsm.Ip, shareInfo), nil
+}
+
+func (service *DsmService) createSMBVolumeByVolume(dsm *webapi.DSM, spec *models.CreateK8sVolumeSpec, srcShareInfo webapi.ShareInfo) (*models.K8sVolumeRespSpec, error) {
+	newSizeInMB := utils.BytesToMBCeil(spec.Size)
+	if spec.Size != 0 && newSizeInMB != srcShareInfo.QuotaValueInMB {
+		return nil,
+			status.Errorf(codes.OutOfRange, "Requested share quotaMB [%d] is not equal to src share quotaMB [%d]", newSizeInMB, srcShareInfo.QuotaValueInMB)
+	}
+
+	shareCloneSpec := webapi.ShareCloneSpec{
+		Name: spec.ShareName,
+		Snapshot: "",
+		ShareInfo: webapi.ShareInfo{
+			Name:                spec.ShareName,
+			VolPath:             srcShareInfo.VolPath, // must be same with srcShare location
+			Desc:                "Cloned from [" + srcShareInfo.Name + "] by csi driver", // max: 64
+			EnableRecycleBin:    srcShareInfo.EnableRecycleBin,
+			RecycleBinAdminOnly: srcShareInfo.RecycleBinAdminOnly,
+			NameOrg:             srcShareInfo.Name,
+		},
+	}
+
+	if _, err := dsm.ShareClone(shareCloneSpec); err != nil && !errors.Is(err, utils.AlreadyExistError("")) {
+		return nil,
+			status.Errorf(codes.Internal, fmt.Sprintf("Failed to create volume with source volume ID: %s, err: %v", srcShareInfo.Uuid, err))
+	}
+
+	shareInfo, err := dsm.ShareGet(spec.ShareName)
+	if err != nil {
+		return nil,
+			status.Errorf(codes.Internal, fmt.Sprintf("Failed to get existed Share with name: [%s], err: %v", spec.ShareName, err))
+	}
+
+	if shareInfo.QuotaValueInMB == 0 {
+		// known issue for some DS, manually set quota to the new share
+		if err := dsm.SetShareQuota(shareInfo, newSizeInMB); err != nil {
+			msg := fmt.Sprintf("Failed to set quota [%d] to Share [%s], err: %v", newSizeInMB, shareInfo.Name, err)
+			log.Error(msg)
+			return nil, status.Errorf(codes.Internal, msg)
+		}
+
+		shareInfo.QuotaValueInMB = newSizeInMB
+	}
+
+	log.Debugf("[%s] createSMBVolumeByVolume Successfully. VolumeId: %s", dsm.Ip, shareInfo.Uuid);
+
+	return DsmShareToK8sVolume(dsm.Ip, shareInfo), nil
+}
+
+func (service *DsmService) createSMBVolumeByDsm(dsm *webapi.DSM, spec *models.CreateK8sVolumeSpec) (*models.K8sVolumeRespSpec, error) {
+	// TODO: Check if share name is allowable
+
+	// 1. Find a available location
+	if spec.Location == "" {
+		vol, err := service.getFirstAvailableVolume(dsm, spec.Size)
+		if err != nil {
+			return nil, status.Errorf(codes.Internal, fmt.Sprintf("Failed to get available location, err: %v", err))
+		}
+		spec.Location = vol.Path
+	}
+
+	// 2. Check if location exists
+	_, err := dsm.VolumeGet(spec.Location)
+	if err != nil {
+		return nil, status.Errorf(codes.InvalidArgument, fmt.Sprintf("Unable to find location %s", spec.Location))
+	}
+
+	// 3. Create Share
+	sizeInMB := utils.BytesToMBCeil(spec.Size)
+	shareSpec := webapi.ShareCreateSpec{
+		Name: spec.ShareName,
+		ShareInfo: webapi.ShareInfo{
+			Name:                spec.ShareName,
+			VolPath:             spec.Location,
+			Desc:                "Created by Synology K8s CSI",
+			EnableShareCow:      false,
+			EnableRecycleBin:    true,
+			RecycleBinAdminOnly: true,
+			Encryption:          0,
+			QuotaForCreate:      &sizeInMB,
+		},
+	}
+
+	log.Debugf("ShareCreate spec: %v", shareSpec)
+	err = dsm.ShareCreate(shareSpec)
+	if err != nil && !errors.Is(err, utils.AlreadyExistError("")) {
+		return nil, status.Errorf(codes.Internal, fmt.Sprintf("Failed to create share, err: %v", err))
+	}
+
+	shareInfo, err := dsm.ShareGet(spec.ShareName)
+	if err != nil {
+		return nil,
+			status.Errorf(codes.Internal, fmt.Sprintf("Failed to get existed Share with name: %s, err: %v", spec.ShareName, err))
+	}
+
+	log.Debugf("[%s] createSMBVolumeByDsm Successfully. VolumeId: %s", dsm.Ip, shareInfo.Uuid)
+
+	return DsmShareToK8sVolume(dsm.Ip, shareInfo), nil
+}
+
+func (service *DsmService) listSMBVolumes(dsmIp string) (infos []*models.K8sVolumeRespSpec) {
+	for _, dsm := range service.dsms {
+		if dsmIp != "" && dsmIp != dsm.Ip {
+			continue
+		}
+
+		shares, err := dsm.ShareList()
+		if err != nil {
+			log.Errorf("[%s] Failed to list shares: %v", dsm.Ip, err)
+			continue
+		}
+
+		for _, share := range shares {
+			if !strings.HasPrefix(share.Name, models.SharePrefix) {
+				continue
+			}
+			infos = append(infos, DsmShareToK8sVolume(dsm.Ip, share))
+		}
+	}
+
+	return infos
+}
+
+func (service *DsmService) listSMBSnapshotsByDsm(dsm *webapi.DSM) (infos []*models.K8sSnapshotRespSpec) {
+	volumes := service.listSMBVolumes(dsm.Ip)
+	for _, volume := range volumes {
+		shareInfo := volume.Share
+		shareSnaps, err := dsm.ShareSnapshotList(shareInfo.Name)
+		if err != nil {
+			log.Errorf("[%s] Failed to list share snapshots: %v", dsm.Ip, err)
+			continue
+		}
+		for _, info := range shareSnaps {
+			infos = append(infos, DsmShareSnapshotToK8sSnapshot(dsm.Ip, info, shareInfo))
+		}
+	}
+	return infos
+}
+
+func (service *DsmService) getSMBSnapshot(snapshotUuid string) *models.K8sSnapshotRespSpec {
+	for _, dsm := range service.dsms {
+		snapshots := service.listSMBSnapshotsByDsm(dsm)
+		for _, snap := range snapshots {
+			if snap.Uuid == snapshotUuid {
+				return snap
+			}
+		}
+	}
+
+	return nil
+}

pkg/dsm/webapi/iscsi.go

@@ -58,6 +58,7 @@ type SnapshotInfo struct {
 	Status            string             `json:"status"`
 	TotalSize         int64              `json:"total_size"`
 	CreateTime        int64              `json:"create_time"`
+	RootPath          string             `json:"root_path"`
 }
 
 type LunDevAttrib struct {
@@ -107,6 +108,8 @@ func errCodeMapping(errCode int, oriErr error) error {
 	switch errCode {
 	case 18990002: // Out of free space
 		return utils.OutOfFreeSpaceError("")
+	case 18990531: // No such LUN
+		return utils.NoSuchLunError("")
 	case 18990538: // Duplicated LUN name
 		return utils.AlreadyExistError("")
 	case 18990541:
@@ -124,7 +127,7 @@ func errCodeMapping(errCode int, oriErr error) error {
 	}
 
 	if errCode > 18990000 {
-		return utils.IscsiDefaultError("")
+		return utils.IscsiDefaultError{errCode}
 	}
 	return oriErr
 }
@@ -143,7 +146,7 @@ func (dsm *DSM) LunList() ([]LunInfo, error) {
 
 	resp, err := dsm.sendRequest("", &LunInfos{}, params, "webapi/entry.cgi")
 	if err != nil {
-		return nil, err
+		return nil, errCodeMapping(resp.ErrorCode, err)
 	}
 
 	lunInfos, ok := resp.Data.(*LunInfos)
@@ -174,9 +177,8 @@ func (dsm *DSM) LunCreate(spec LunCreateSpec) (string, error) {
 	}
 
 	resp, err := dsm.sendRequest("", &LunCreateResp{}, params, "webapi/entry.cgi")
-	err = errCodeMapping(resp.ErrorCode, err)
 	if err != nil {
-		return "", err
+		return "", errCodeMapping(resp.ErrorCode, err)
 	}
 
 	lunResp, ok := resp.Data.(*LunCreateResp)
@@ -195,9 +197,9 @@ func (dsm *DSM) LunUpdate(spec LunUpdateSpec) error {
 	params.Add("uuid", strconv.Quote(spec.Uuid))
 	params.Add("new_size", strconv.FormatInt(int64(spec.NewSize), 10))
 
-	_, err := dsm.sendRequest("", &struct{}{}, params, "webapi/entry.cgi")
+	resp, err := dsm.sendRequest("", &struct{}{}, params, "webapi/entry.cgi")
 	if err != nil {
-		return err
+		return errCodeMapping(resp.ErrorCode, err)
 	}
 
 	return nil
@@ -216,9 +218,9 @@ func (dsm *DSM) LunGet(uuid string) (LunInfo, error) {
 	}
 	info := Info{}
 
-	_, err := dsm.sendRequest("", &info, params, "webapi/entry.cgi")
+	resp, err := dsm.sendRequest("", &info, params, "webapi/entry.cgi")
 	if err != nil {
-		return LunInfo{}, err
+		return LunInfo{}, errCodeMapping(resp.ErrorCode, err)
 	}
 
 	return info.Lun, nil
@@ -238,9 +240,8 @@ func (dsm *DSM) LunClone(spec LunCloneSpec) (string, error) {
 	}
 
 	resp, err := dsm.sendRequest("", &LunCloneResp{}, params, "webapi/entry.cgi")
-	err = errCodeMapping(resp.ErrorCode, err)
 	if err != nil {
-		return "", err
+		return "", errCodeMapping(resp.ErrorCode, err)
 	}
 
 	cloneLunResp, ok := resp.Data.(*LunCloneResp)
@@ -264,7 +265,7 @@ func (dsm *DSM) TargetList() ([]TargetInfo, error) {
 
 	resp, err := dsm.sendRequest("", &TargetInfos{}, params, "webapi/entry.cgi")
 	if err != nil {
-		return nil, err
+		return nil, errCodeMapping(resp.ErrorCode, err)
 	}
 
 	trgInfos, ok := resp.Data.(*TargetInfos)
@@ -287,9 +288,9 @@ func (dsm *DSM) TargetGet(targetId string) (TargetInfo, error) {
 	}
 	info := Info{}
 
-	_, err := dsm.sendRequest("", &info, params, "webapi/entry.cgi")
+	resp, err := dsm.sendRequest("", &info, params, "webapi/entry.cgi")
 	if err != nil {
-		return TargetInfo{}, err
+		return TargetInfo{}, errCodeMapping(resp.ErrorCode, err)
 	}
 
 	return info.Target, nil
@@ -304,9 +305,9 @@ func (dsm *DSM) TargetSet(targetId string, maxSession int) error {
 	params.Add("target_id", strconv.Quote(targetId))
 	params.Add("max_sessions", strconv.Itoa(maxSession))
 
-	_, err := dsm.sendRequest("", &struct{}{}, params, "webapi/entry.cgi")
+	resp, err := dsm.sendRequest("", &struct{}{}, params, "webapi/entry.cgi")
 	if err != nil {
-		return err
+		return errCodeMapping(resp.ErrorCode, err)
 	}
 
 	return nil
@@ -326,9 +327,8 @@ func (dsm *DSM) TargetCreate(spec TargetCreateSpec) (string, error) {
 	}
 
 	resp, err := dsm.sendRequest("", &TrgCreateResp{}, params, "webapi/entry.cgi")
-	err = errCodeMapping(resp.ErrorCode, err)
 	if err != nil {
-		return "", err
+		return "", errCodeMapping(resp.ErrorCode, err)
 	}
 
 	trgResp, ok := resp.Data.(*TrgCreateResp)
@@ -351,9 +351,9 @@ func (dsm *DSM) LunMapTarget(targetIds []string, lunUuid string) error {
 		log.Debugln(params)
 	}
 
-	_, err := dsm.sendRequest("", &struct{}{}, params, "webapi/entry.cgi")
+	resp, err := dsm.sendRequest("", &struct{}{}, params, "webapi/entry.cgi")
 	if err != nil {
-		return err
+		return errCodeMapping(resp.ErrorCode, err)
 	}
 	return nil
 }
@@ -365,9 +365,9 @@ func (dsm *DSM) LunDelete(lunUuid string) error {
 	params.Add("version", "1")
 	params.Add("uuid", strconv.Quote(lunUuid))
 
-	_, err := dsm.sendRequest("", &struct{}{}, params, "webapi/entry.cgi")
+	resp, err := dsm.sendRequest("", &struct{}{}, params, "webapi/entry.cgi")
 	if err != nil {
-		return err
+		return errCodeMapping(resp.ErrorCode, err)
 	}
 	return nil
 }
@@ -379,9 +379,9 @@ func (dsm *DSM) TargetDelete(targetName string) error {
 	params.Add("version", "1")
 	params.Add("target_id", strconv.Quote(targetName))
 
-	_, err := dsm.sendRequest("", &struct{}{}, params, "webapi/entry.cgi")
+	resp, err := dsm.sendRequest("", &struct{}{}, params, "webapi/entry.cgi")
 	if err != nil {
-		return err
+		return errCodeMapping(resp.ErrorCode, err)
 	}
 	return nil
 }

pkg/dsm/webapi/share.go

@@ -7,21 +7,109 @@ import (
 	"fmt"
 	"net/url"
 	"strconv"
+
+	log "github.com/sirupsen/logrus"
+	"github.com/SynologyOpenSource/synology-csi/pkg/utils"
+	"github.com/SynologyOpenSource/synology-csi/pkg/logger"
 )
 
 type ShareInfo struct {
-	Name                string `json:"name"`
-	VolPath             string `json:"vol_path"`
+	Name                string `json:"name"`                        // required
+	VolPath             string `json:"vol_path"`                    // required
 	Desc                string `json:"desc"`
-	EnableShareCow      bool   `json:"enable_share_cow"`
+	EnableShareCow      bool   `json:"enable_share_cow"`            // field for create
 	EnableRecycleBin    bool   `json:"enable_recycle_bin"`
 	RecycleBinAdminOnly bool   `json:"recycle_bin_admin_only"`
-	Encryption          int    `json:"encryption"`
+	Encryption          int    `json:"encryption"`                  // field for create
+	QuotaForCreate      *int64 `json:"share_quota,omitempty"`
+	QuotaValueInMB      int64  `json:"quota_value"`                 // field for get
+	SupportSnapshot     bool   `json:"support_snapshot"`            // field for get
+	Uuid                string `json:"uuid"`                        // field for get
+	NameOrg             string `json:"name_org"`                    // required for clone
+}
+
+type ShareUpdateInfo struct {
+	Name                string `json:"name"`                        // required
+	VolPath             string `json:"vol_path"`                    // required
+	QuotaForCreate      *int64 `json:"share_quota,omitempty"`
+	// Add properties you want to update to shares here
+}
+
+type ShareSnapshotInfo struct {
+	Uuid             string `json:"ruuid"`
+	Time             string `json:"time"`
+	Desc             string `json:"desc"`
+	SnapSize         string `json:"snap_size"` // the complete size of the snapshot
+	Lock             bool   `json:"lock"`
+	ScheduleSnapshot bool   `json:"schedule_snapshot"`
 }
 
 type ShareCreateSpec struct {
-	Name      string    `json:"name"`
-	ShareInfo ShareInfo `json:"shareinfo"`
+	Name      string
+	ShareInfo ShareInfo
+}
+
+type ShareCloneSpec struct {
+	Name      string
+	Snapshot  string
+	ShareInfo ShareInfo
+}
+
+type ShareSnapshotCreateSpec struct {
+	ShareName string
+	Desc      string
+	IsLocked  bool
+}
+
+type SharePermissionSetSpec struct {
+	Name          string
+	UserGroupType string            // "local_user"/"local_group"/"system"
+	Permissions   []*SharePermission
+}
+
+type SharePermission struct {
+	Name       string `json:"name"`
+	IsReadonly bool   `json:"is_readonly"`
+	IsWritable bool   `json:"is_writable"`
+	IsDeny     bool   `json:"is_deny"`
+	IsCustom   bool   `json:"is_custom,omitempty"`
+	IsAdmin    bool   `json:"is_admin,omitempty"` // field for list
+}
+
+func shareErrCodeMapping(errCode int, oriErr error) error {
+	switch errCode {
+	case 402: // No such share
+		return utils.NoSuchShareError("")
+	case 403: // Invalid input value
+		return utils.BadParametersError("")
+	case 3301: // already exists
+		return utils.AlreadyExistError("")
+	case 3309:
+		return utils.ShareReachMaxCountError("")
+	case 3328:
+		return utils.ShareSystemBusyError("")
+	}
+
+	if errCode >= 3300 {
+		return utils.ShareDefaultError{errCode}
+	}
+	return oriErr
+}
+
+// ----------------------- Share APIs -----------------------
+func (dsm *DSM) ShareGet(shareName string) (ShareInfo, error) {
+	params := url.Values{}
+	params.Add("api", "SYNO.Core.Share")
+	params.Add("method", "get")
+	params.Add("version", "1")
+	params.Add("additional", "[\"encryption\", \"enable_share_cow\", \"recyclebin\", \"support_snapshot\", \"share_quota\"]")
+	params.Add("name", strconv.Quote(shareName))
+
+	info := ShareInfo{}
+
+	resp, err := dsm.sendRequest("", &info, params, "webapi/entry.cgi")
+
+	return info, shareErrCodeMapping(resp.ErrorCode, err)
 }
 
 func (dsm *DSM) ShareList() ([]ShareInfo, error) {
@@ -29,7 +117,7 @@ func (dsm *DSM) ShareList() ([]ShareInfo, error) {
 	params.Add("api", "SYNO.Core.Share")
 	params.Add("method", "list")
 	params.Add("version", "1")
-	params.Add("additional", "[\"encryption\"]")
+	params.Add("additional", "[\"encryption\", \"enable_share_cow\", \"recyclebin\", \"support_snapshot\", \"share_quota\"]")
 
 	type ShareInfos struct {
 		Shares []ShareInfo `json:"shares"`
@@ -37,7 +125,7 @@ func (dsm *DSM) ShareList() ([]ShareInfo, error) {
 
 	resp, err := dsm.sendRequest("", &ShareInfos{}, params, "webapi/entry.cgi")
 	if err != nil {
-		return nil, err
+		return nil, shareErrCodeMapping(resp.ErrorCode, err)
 	}
 
 	infos, ok := resp.Data.(*ShareInfos)
@@ -61,13 +149,49 @@ func (dsm *DSM) ShareCreate(spec ShareCreateSpec) error {
 	}
 	params.Add("shareinfo", string(js))
 
-	// response : {"data":{"name":"Share-2"},"success":true}
-	_, err = dsm.sendRequest("", &struct{}{}, params, "webapi/entry.cgi")
-	if err != nil {
-		return err
+	resp, err := dsm.sendRequest("", &struct{}{}, params, "webapi/entry.cgi")
+
+	return shareErrCodeMapping(resp.ErrorCode, err)
+}
+
+func (dsm *DSM) ShareClone(spec ShareCloneSpec) (string, error) {
+	params := url.Values{}
+	params.Add("api", "SYNO.Core.Share")
+	params.Add("method", "clone")
+	params.Add("version", "1")
+	params.Add("name", strconv.Quote(spec.Name))
+
+	// if clone from snapshot, the NameOrg must be the parent of the snapshot, or the webapi will return 3300
+	// if the snapshot doesn't exist, it will return 3300 too.
+	if spec.ShareInfo.NameOrg == "" {
+		return "", fmt.Errorf("Clone failed. The source name can't be empty.")
 	}
 
-	return nil
+	if spec.Snapshot != "" {
+		params.Add("snapshot", strconv.Quote(spec.Snapshot))
+	}
+
+	js, err := json.Marshal(spec.ShareInfo)
+	if err != nil {
+		return "", err
+	}
+	params.Add("shareinfo", string(js))
+
+	type ShareCreateResp struct {
+		Name string `json:"name"`
+	}
+
+	resp, err := dsm.sendRequest("", &ShareCreateResp{}, params, "webapi/entry.cgi")
+	if err != nil {
+		return "", shareErrCodeMapping(resp.ErrorCode, err)
+	}
+
+	shareResp, ok := resp.Data.(*ShareCreateResp)
+	if !ok {
+		return "", fmt.Errorf("Failed to assert response to %T", &ShareCreateResp{})
+	}
+
+	return shareResp.Name, nil
 }
 
 func (dsm *DSM) ShareDelete(shareName string) error {
@@ -75,13 +199,165 @@ func (dsm *DSM) ShareDelete(shareName string) error {
 	params.Add("api", "SYNO.Core.Share")
 	params.Add("method", "delete")
 	params.Add("version", "1")
+	params.Add("name", fmt.Sprintf("[%s]", strconv.Quote(shareName)))
+
+	resp, err := dsm.sendRequest("", &struct{}{}, params, "webapi/entry.cgi")
+
+	return shareErrCodeMapping(resp.ErrorCode, err)
+}
+
+func (dsm *DSM) ShareSet(shareName string, updateInfo ShareUpdateInfo) error {
+	params := url.Values{}
+	params.Add("api", "SYNO.Core.Share")
+	params.Add("method", "set")
+	params.Add("version", "1")
 	params.Add("name", strconv.Quote(shareName))
 
-	// response : {"success":true}
-	_, err := dsm.sendRequest("", &struct{}{}, params, "webapi/entry.cgi")
+	js, err := json.Marshal(updateInfo)
 	if err != nil {
 		return err
 	}
+	params.Add("shareinfo", string(js))
+
+	if logger.WebapiDebug {
+		log.Debugln(params)
+	}
+
+	resp, err := dsm.sendRequest("", &struct{}{}, params, "webapi/entry.cgi")
+
+	return shareErrCodeMapping(resp.ErrorCode, err)
+}
+
+func (dsm *DSM) SetShareQuota(shareInfo ShareInfo, newSizeInMB int64) error {
+	updateInfo := ShareUpdateInfo{
+		Name:           shareInfo.Name,
+		VolPath:        shareInfo.VolPath,
+		QuotaForCreate: &newSizeInMB,
+	}
+	return dsm.ShareSet(shareInfo.Name, updateInfo)
+}
+
+// ----------------------- Share Snapshot APIs -----------------------
+func (dsm *DSM) ShareSnapshotCreate(spec ShareSnapshotCreateSpec) (string, error) {
+	params := url.Values{}
+	params.Add("api", "SYNO.Core.Share.Snapshot")
+	params.Add("method", "create")
+	params.Add("version", "1")
+	params.Add("name", strconv.Quote(spec.ShareName))
+
+	type SnapInfo struct {
+		Desc     string `json:"desc"`
+		IsLocked bool   `json:"lock"` // default true
+	}
+
+	snapinfo := SnapInfo{
+		Desc:     spec.Desc,
+		IsLocked: spec.IsLocked,
+	}
+	js, err := json.Marshal(snapinfo)
+	if err != nil {
+		return "", err
+	}
+	params.Add("snapinfo", string(js))
+
+	var snapTime string
+	resp, err := dsm.sendRequest("", &snapTime, params, "webapi/entry.cgi")
+	if err != nil {
+		return "", shareErrCodeMapping(resp.ErrorCode, err)
+	}
+
+	return snapTime, nil // "GMT+08-2022.01.14-19.18.29"
+}
+
+func (dsm *DSM) ShareSnapshotList(name string) ([]ShareSnapshotInfo, error) {
+	params := url.Values{}
+	params.Add("api", "SYNO.Core.Share.Snapshot")
+	params.Add("method", "list")
+	params.Add("version", "2")
+	params.Add("name", strconv.Quote(name))
+	params.Add("additional", "[\"desc\", \"lock\", \"schedule_snapshot\", \"ruuid\", \"snap_size\"]")
+
+	type Infos struct {
+		Snapshots []ShareSnapshotInfo `json:"snapshots"`
+		Total     int                 `json:"total"`
+	}
+
+	resp, err := dsm.sendRequest("", &Infos{}, params, "webapi/entry.cgi")
+	if err != nil {
+		return nil, shareErrCodeMapping(resp.ErrorCode, err)
+	}
+
+	infos, ok := resp.Data.(*Infos)
+	if !ok {
+		return nil, fmt.Errorf("Failed to assert response to %T", &Infos{})
+	}
+
+	return infos.Snapshots, nil
+}
+
+func (dsm *DSM) ShareSnapshotDelete(snapTime string, shareName string) error {
+	params := url.Values{}
+	params.Add("api", "SYNO.Core.Share.Snapshot")
+	params.Add("method", "delete")
+	params.Add("version", "1")
+	params.Add("name", strconv.Quote(shareName))
+	params.Add("snapshots", fmt.Sprintf("[%s]", strconv.Quote(snapTime))) // ["GMT+08-2022.01.14-19.18.29"]
+
+	var objmap []map[string]interface{}
+	resp, err := dsm.sendRequest("", &objmap, params, "webapi/entry.cgi")
+	if err != nil {
+		return shareErrCodeMapping(resp.ErrorCode, err)
+	}
+
+	if len(objmap) > 0 {
+		return fmt.Errorf("Failed to delete snapshot, API common error. snapshot: %s", snapTime)
+	}
 
 	return nil
 }
+
+// ----------------------- Share Permission APIs -----------------------
+func (dsm *DSM) SharePermissionSet(spec SharePermissionSetSpec) error {
+	params := url.Values{}
+	params.Add("api", "SYNO.Core.Share.Permission")
+	params.Add("method", "set")
+	params.Add("version", "1")
+	params.Add("name", strconv.Quote(spec.Name))
+	params.Add("user_group_type", strconv.Quote(spec.UserGroupType))
+
+	js, err := json.Marshal(spec.Permissions)
+	if err != nil {
+		return err
+	}
+	params.Add("permissions", string(js))
+
+	resp, err := dsm.sendRequest("", &struct{}{}, params, "webapi/entry.cgi")
+
+	return shareErrCodeMapping(resp.ErrorCode, err)
+}
+
+func (dsm *DSM) SharePermissionList(shareName string, userGroupType string) ([]SharePermission, error) {
+	params := url.Values{}
+	params.Add("api", "SYNO.Core.Share.Permission")
+	params.Add("method", "list")
+	params.Add("version", "1")
+	params.Add("name", strconv.Quote(shareName))
+	params.Add("user_group_type", strconv.Quote(userGroupType))
+
+	type SharePermissions struct {
+		Permissions []SharePermission `json:"items"`
+	}
+
+	resp, err := dsm.sendRequest("", &SharePermissions{}, params, "webapi/entry.cgi")
+	if err != nil {
+		return nil, shareErrCodeMapping(resp.ErrorCode, err)
+	}
+
+	infos, ok := resp.Data.(*SharePermissions)
+	if !ok {
+		return nil, fmt.Errorf("Failed to assert response to %T", &SharePermissions{})
+	}
+
+	return infos.Permissions, nil
+}
+

pkg/interfaces/IDsmService.go

@@ -16,14 +16,15 @@ type IDsmService interface {
 	GetDsm(ip string) (*webapi.DSM, error)
 	GetDsmsCount() int
 	ListDsmVolumes(ip string) ([]webapi.VolInfo, error)
-	CreateVolume(spec *models.CreateK8sVolumeSpec) (webapi.LunInfo, string, error)
-	DeleteVolume(volumeId string) error
-	ListVolumes() []*models.ListK8sVolumeRespSpec
-	GetVolume(lunUuid string) *models.ListK8sVolumeRespSpec
-	ExpandLun(lunUuid string, newSize int64) error
-	CreateSnapshot(spec *models.CreateK8sVolumeSnapshotSpec) (string, error)
+	CreateVolume(spec *models.CreateK8sVolumeSpec) (*models.K8sVolumeRespSpec, error)
+	DeleteVolume(volId string) error
+	ListVolumes() []*models.K8sVolumeRespSpec
+	GetVolume(volId string) *models.K8sVolumeRespSpec
+	ExpandVolume(volId string, newSize int64) (*models.K8sVolumeRespSpec, error)
+	CreateSnapshot(spec *models.CreateK8sVolumeSnapshotSpec) (*models.K8sSnapshotRespSpec, error)
 	DeleteSnapshot(snapshotUuid string) error
-	ListAllSnapshots() ([]webapi.SnapshotInfo, error)
-	ListSnapshots(lunUuid string) ([]webapi.SnapshotInfo, error)
-	GetSnapshot(snapshotUuid string) (webapi.SnapshotInfo, error)
+	ListAllSnapshots() []*models.K8sSnapshotRespSpec
+	ListSnapshots(volId string) []*models.K8sSnapshotRespSpec
+	GetVolumeByName(volName string) *models.K8sVolumeRespSpec
+	GetSnapshotByName(snapshotName string) *models.K8sSnapshotRespSpec
 }

pkg/models/dsm.go

@@ -2,6 +2,10 @@
 
 package models
 
+import (
+	"fmt"
+)
+
 const (
 	K8sCsiName       = "Kubernetes CSI"
 
@@ -15,8 +19,30 @@ const (
 	LunTypeBlunThick = "BLUN_THICK"         // thick provision, mapped to type 259
 	MaxIqnLen = 128
 
+	// Share definitions
+	MaxShareLen     = 32
+	MaxShareDescLen = 64
+	UserGroupTypeLocalUser  = "local_user"
+	UserGroupTypeLocalGroup = "local_group"
+	UserGroupTypeSystem     = "system"
+
+
 	// CSI definitions
-	TargetPrefix           = "k8s-csi"
-	LunPrefix              = "k8s-csi"
-	IqnPrefix              = "iqn.2000-01.com.synology:"
-)
+	TargetPrefix            = "k8s-csi"
+	LunPrefix               = "k8s-csi"
+	IqnPrefix               = "iqn.2000-01.com.synology:"
+	SharePrefix             = "k8s-csi"
+	ShareSnapshotDescPrefix = "(Do not change)"
+)
+
+func GenLunName(volName string) string {
+	return fmt.Sprintf("%s-%s", LunPrefix, volName)
+}
+
+func GenShareName(volName string) string {
+	shareName := fmt.Sprintf("%s-%s", SharePrefix, volName)
+	if len(shareName) > MaxShareLen {
+		return shareName[:MaxShareLen]
+	}
+	return shareName
+}

pkg/models/dsm_req_spec.go

@@ -3,6 +3,8 @@
 package models
 
 import (
+	"fmt"
+	"github.com/container-storage-interface/spec/lib/go/csi"
 	"github.com/SynologyOpenSource/synology-csi/pkg/dsm/webapi"
 )
 
@@ -10,21 +12,43 @@ type CreateK8sVolumeSpec struct {
 	DsmIp            string
 	K8sVolumeName    string
 	LunName          string
+	ShareName        string
 	Location         string
 	Size             int64
 	Type             string
 	ThinProvisioning bool
 	TargetName       string
-	TargetIqn        string
 	MultipleSession  bool
 	SourceSnapshotId string
 	SourceVolumeId   string
+	Protocol         string
 }
 
-type ListK8sVolumeRespSpec struct {
-	DsmIp            string
-	Lun              webapi.LunInfo
-	Target           webapi.TargetInfo
+type K8sVolumeRespSpec struct {
+	DsmIp             string
+	VolumeId          string
+	SizeInBytes       int64
+	Location          string
+	Name              string
+	Source            string
+	Lun               webapi.LunInfo
+	Target            webapi.TargetInfo
+	Share             webapi.ShareInfo
+	Protocol          string
+}
+
+type K8sSnapshotRespSpec struct {
+	DsmIp             string
+	Name              string
+	Uuid              string
+	ParentName        string
+	ParentUuid        string
+	Status            string
+	SizeInBytes       int64
+	CreateTime        int64
+	Time              string // only for share snapshot delete
+	RootPath          string
+	Protocol          string
 }
 
 type CreateK8sVolumeSnapshotSpec struct {
@@ -33,4 +57,24 @@ type CreateK8sVolumeSnapshotSpec struct {
 	Description  string
 	TakenBy      string
 	IsLocked     bool
-}
+}
+
+type NodeStageVolumeSpec struct {
+	VolumeId          string
+	StagingTargetPath string
+	VolumeCapability  *csi.VolumeCapability
+	Dsm               string
+	Source            string
+}
+
+type ByVolumeId []*K8sVolumeRespSpec
+func (a ByVolumeId) Len() int           { return len(a) }
+func (a ByVolumeId) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
+func (a ByVolumeId) Less(i, j int) bool { return a[i].VolumeId < a[j].VolumeId }
+
+type BySnapshotAndParentUuid []*K8sSnapshotRespSpec
+func (a BySnapshotAndParentUuid) Len() int           { return len(a) }
+func (a BySnapshotAndParentUuid) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
+func (a BySnapshotAndParentUuid) Less(i, j int) bool {
+	return fmt.Sprintf("%s/%s", a[i].ParentUuid, a[i].Uuid) < fmt.Sprintf("%s/%s", a[j].ParentUuid, a[j].Uuid)
+}

pkg/utils/error.go

@@ -2,14 +2,28 @@
 
 package utils
 
+import (
+	"fmt"
+)
+
 type OutOfFreeSpaceError string
 type AlreadyExistError string
+type BadParametersError string
+type NoSuchLunError string
 type LunReachMaxCountError string
 type TargetReachMaxCountError string
 type NoSuchSnapshotError string
 type BadLunTypeError string
 type SnapshotReachMaxCountError string
-type IscsiDefaultError string
+type IscsiDefaultError struct {
+	ErrCode int
+}
+type NoSuchShareError string
+type ShareReachMaxCountError string
+type ShareSystemBusyError string
+type ShareDefaultError struct {
+	ErrCode int
+}
 
 func (_ OutOfFreeSpaceError) Error() string {
 	return "Out of free space"
@@ -17,6 +31,14 @@ func (_ OutOfFreeSpaceError) Error() string {
 func (_ AlreadyExistError) Error() string {
 	return "Already Existed"
 }
+func (_ BadParametersError) Error() string {
+	return "Invalid input value"
+}
+
+// ISCSI errors
+func (_ NoSuchLunError) Error() string {
+	return "No such LUN"
+}
 
 func (_ LunReachMaxCountError) Error() string {
 	return "Number of LUN reach limit"
@@ -38,6 +60,23 @@ func (_ SnapshotReachMaxCountError) Error() string {
 	return "Number of snapshot reach limit"
 }
 
-func (_ IscsiDefaultError) Error() string {
-	return "Default ISCSI error"
-}
+func (e IscsiDefaultError) Error() string {
+	return fmt.Sprintf("ISCSI API error. Error code: %d", e.ErrCode)
+}
+
+// Share errors
+func (_ NoSuchShareError) Error() string {
+	return "No such share"
+}
+
+func (_ ShareReachMaxCountError) Error() string {
+	return "Number of share reach limit"
+}
+
+func (_ ShareSystemBusyError) Error() string {
+	return "Share system is temporary busy"
+}
+
+func (e ShareDefaultError) Error() string {
+	return fmt.Sprintf("Share API error. Error code: %d", e.ErrCode)
+}

pkg/utils/utils.go

@@ -8,7 +8,42 @@ import (
 	"strings"
 )
 
-const UNIT_GB = 1024 * 1024 * 1024
+type AuthType string
+
+const (
+	UNIT_GB = 1024 * 1024 * 1024
+	UNIT_MB = 1024 * 1024
+
+	ProtocolSmb     = "smb"
+	ProtocolIscsi   = "iscsi"
+	ProtocolDefault = ProtocolIscsi
+
+	AuthTypeReadWrite AuthType = "rw"
+	AuthTypeReadOnly  AuthType = "ro"
+	AuthTypeNoAccess  AuthType = "no"
+)
+
+func SliceContains(items []string, s string) bool {
+	for _, item := range items {
+		if s == item {
+			return true
+		}
+	}
+	return false
+}
+
+func MBToBytes(size int64) int64 {
+	return size * UNIT_MB
+}
+
+func BytesToMB(size int64) int64 {
+	return size / UNIT_MB
+}
+
+// Ceiling
+func BytesToMBCeil(size int64) int64 {
+	return (size + UNIT_MB - 1) / UNIT_MB
+}
 
 func StringToBoolean(value string) bool {
 	value = strings.ToLower(value)
@@ -30,4 +65,4 @@ func LookupIPv4(name string) ([]string, error) {
 	}
 
 	return nil, fmt.Errorf("Failed to LookupIPv4 by local resolver for: %s", name)
-}
+}

scripts/deploy.sh

@@ -1,12 +1,39 @@
 #!/bin/bash
 plugin_name="csi.san.synology.com"
-deploy_k8s_version="v1.19"
+min_support_minor=19
+max_support_minor=20
+deploy_k8s_version="v1".$min_support_minor
 
 SCRIPT_PATH="$(realpath "$0")"
 SOURCE_PATH="$(realpath "$(dirname "${SCRIPT_PATH}")"/../)"
 config_file="${SOURCE_PATH}/config/client-info.yml"
 plugin_dir="/var/lib/kubelet/plugins/$plugin_name"
 
+parse_version(){
+    ver=$(kubectl version --short | grep Server | awk '{print $3}')
+    major=$(echo "${ver##*v}" | cut -d'.' -f1)
+    minor=$(echo "${ver##*v}" | cut -d'.' -f2)
+
+    if [[ "$major" != 1 ]]; then
+        echo "Version not supported: $ver"
+        exit 1
+    fi
+
+    case "$minor" in
+        19|20)
+            deploy_k8s_version="v1".$minor
+            ;;
+        *)
+            if [[ $minor -lt $min_support_minor ]]; then
+                deploy_k8s_version="v1".$min_support_minor
+            else
+                deploy_k8s_version="v1".$max_support_minor
+            fi
+            ;;
+    esac
+    echo "Deploy Version: $deploy_k8s_version"
+}
+
 # 1. Build
 csi_build(){
     echo "==== Build synology-csi .... ===="
@@ -16,6 +43,7 @@ csi_build(){
 # 2. Install
 csi_install(){
     echo "==== Creates namespace and secrets, then installs synology-csi ===="
+    parse_version
 
     kubectl create ns synology-csi
     kubectl create secret -n synology-csi generic client-info-secret --from-file="$config_file"

synocli/cmd/dsm.go

@@ -7,6 +7,7 @@ import (
 	"fmt"
 	"os"
 	"github.com/spf13/cobra"
+	"github.com/SynologyOpenSource/synology-csi/pkg/dsm/common"
 	"github.com/SynologyOpenSource/synology-csi/pkg/dsm/webapi"
 )
 
@@ -51,6 +52,30 @@ var cmdDsmLogin = &cobra.Command{
 	},
 }
 
+// Always get the first client from ClientInfo for synocli testing
+func LoginDsmForTest() (*webapi.DSM, error) {
+	info, err := common.LoadConfig("./config/client-info.yml")
+	if err != nil {
+		return nil, fmt.Errorf("Failed to read config: %v", err)
+	}
+	if len(info.Clients) == 0 {
+		return nil, fmt.Errorf("No client in client-info.yml")
+	}
+
+	dsm := &webapi.DSM{
+		Ip:       info.Clients[0].Host,
+		Port:     info.Clients[0].Port,
+		Username: info.Clients[0].Username,
+		Password: info.Clients[0].Password,
+		Https:    info.Clients[0].Https,
+	}
+
+	if err := dsm.Login(); err != nil {
+		return nil, fmt.Errorf("Failed to login to DSM: [%s]. err: %v", dsm.Ip, err)
+	}
+	return dsm, nil
+}
+
 func init() {
 	cmdDsm.AddCommand(cmdDsmLogin)
 

synocli/cmd/root.go

@@ -16,6 +16,7 @@ var rootCmd = &cobra.Command{
 
 func init() {
 	rootCmd.AddCommand(cmdDsm)
+	rootCmd.AddCommand(cmdShare)
 }
 
 func Execute() {

synocli/cmd/share.go

@@ -0,0 +1,580 @@
+/*
+ * Copyright 2022 Synology Inc.
+ */
+package cmd
+
+import (
+	"fmt"
+	"os"
+	"strings"
+	"github.com/spf13/cobra"
+	"github.com/SynologyOpenSource/synology-csi/pkg/dsm/webapi"
+	"github.com/SynologyOpenSource/synology-csi/pkg/utils"
+	"strconv"
+
+	"text/tabwriter"
+)
+
+var cmdShare = &cobra.Command{
+	Use:   "share",
+	Short: "share API",
+	Long:  `DSM share API`,
+	Run: func(cmd *cobra.Command, args []string) {
+		cmd.Help()
+	},
+}
+
+var cmdShareGet = &cobra.Command{
+	Use:   "get <name>",
+	Short: "get share",
+	Args:  cobra.MinimumNArgs(1),
+	Run: func(cmd *cobra.Command, args []string) {
+		dsm, err := LoginDsmForTest()
+		if err != nil {
+			fmt.Println(err)
+			os.Exit(1)
+		}
+		defer func() {
+			dsm.Logout()
+		}()
+
+		share, err := dsm.ShareGet(args[0])
+		if err != nil {
+			fmt.Println(err)
+			os.Exit(1)
+		}
+		fmt.Printf("Success, ShareGet(%s) = %#v\n", args[0], share)
+	},
+}
+
+var cmdShareList = &cobra.Command{
+	Use:   "list",
+	Short: "list shares",
+	Args:  cobra.MinimumNArgs(0),
+	Run: func(cmd *cobra.Command, args []string) {
+		dsm, err := LoginDsmForTest()
+		if err != nil {
+			fmt.Println(err)
+			os.Exit(1)
+		}
+		defer func() {
+			dsm.Logout()
+		}()
+
+		shares, err := dsm.ShareList()
+		if err != nil {
+			fmt.Println(err)
+			os.Exit(1)
+		}
+
+		tw := tabwriter.NewWriter(os.Stdout, 8, 0, 2, ' ', 0)
+
+		fmt.Fprintf(tw, "%s\t%-32s\t%-10s\t%-64s\t%s\t%s\t%s\t%s\t%-10s\t%s\t%s\n",
+			"id:", "Name:", "VolPath", "Desc:", "Quota(MB):", "ShareCow:", "RecycleBin:", "RBAdminOnly:", "Encryption:", "CanSnap:", "Uuid:")
+		for i, info := range shares {
+			fmt.Fprintf(tw, "%d\t%-32s\t", i, info.Name)
+			fmt.Fprintf(tw, "%-10s\t", info.VolPath)
+			fmt.Fprintf(tw, "%-64s\t", info.Desc)
+			fmt.Fprintf(tw, "%d\t", info.QuotaValueInMB)
+			fmt.Fprintf(tw, "%v\t", info.EnableShareCow)
+			fmt.Fprintf(tw, "%v\t", info.EnableRecycleBin)
+			fmt.Fprintf(tw, "%v\t", info.RecycleBinAdminOnly)
+			fmt.Fprintf(tw, "%-10d\t", info.Encryption)
+			fmt.Fprintf(tw, "%v\t", info.SupportSnapshot)
+			fmt.Fprintf(tw, "%s\t", info.Uuid)
+			fmt.Fprintf(tw, "\n")
+
+			_ = tw.Flush()
+		}
+
+		fmt.Printf("Success, ShareList()\n")
+	},
+}
+
+var cmdShareCreate = &cobra.Command{
+	Use:   "create <name> <location> [<size_bytes>]",
+	Short: "create share",
+	Args:  cobra.MinimumNArgs(2),
+	Run: func(cmd *cobra.Command, args []string) {
+		dsm, err := LoginDsmForTest()
+		if err != nil {
+			fmt.Println(err)
+			os.Exit(1)
+		}
+		defer func() {
+			dsm.Logout()
+		}()
+
+		var size int64 = 0
+		if len(args) >= 3 {
+			size, err = strconv.ParseInt(args[2], 10, 64)
+			if err != nil {
+			    fmt.Println(err)
+				os.Exit(1)
+			}
+		}
+		sizeInMB := utils.BytesToMBCeil(size)
+		testSpec := webapi.ShareCreateSpec{
+			Name: args[0],
+			ShareInfo: webapi.ShareInfo{
+				Name:                args[0],
+				VolPath:             args[1],
+				Desc:                "Created by synocli",
+				EnableShareCow:      false,
+				EnableRecycleBin:    true,
+				RecycleBinAdminOnly: true,
+				Encryption:          0,
+				QuotaForCreate:      &sizeInMB,
+			},
+		}
+
+		fmt.Printf("spec = %#v, sizeInMB = %d\n", testSpec, sizeInMB)
+		err = dsm.ShareCreate(testSpec)
+		if err != nil {
+			fmt.Println(err)
+			os.Exit(1)
+		}
+
+		share, err := dsm.ShareGet(args[0])
+		if err != nil {
+			fmt.Println(err)
+			os.Exit(1)
+		}
+
+		fmt.Printf("Success, ShareCreate(%s) resp = %#v\n", args[0], share)
+	},
+}
+
+var cmdShareDelete = &cobra.Command{
+	Use:   "delete <name>",
+	Short: "delete share",
+	Args:  cobra.MinimumNArgs(1),
+	Run: func(cmd *cobra.Command, args []string) {
+		dsm, err := LoginDsmForTest()
+		if err != nil {
+			fmt.Println(err)
+			os.Exit(1)
+		}
+		defer func() {
+			dsm.Logout()
+		}()
+
+		err = dsm.ShareDelete(args[0])
+		if err != nil {
+			fmt.Println(err)
+			os.Exit(1)
+		}
+		fmt.Printf("Success, ShareDelete(%s)\n", args[0])
+	},
+}
+
+var cmdShareClone = &cobra.Command{
+	Use:   "clone <new name> <src name> <is_snapshot [true|false]>",
+	Short: "clone share",
+	Args:  cobra.MinimumNArgs(2),
+	Run: func(cmd *cobra.Command, args []string) {
+		dsm, err := LoginDsmForTest()
+		if err != nil {
+			fmt.Println(err)
+			os.Exit(1)
+		}
+		defer func() {
+			dsm.Logout()
+		}()
+
+		fromSnapshot := false
+		if len(args) >= 3 && args[2] == "true" {
+			fromSnapshot = true
+		}
+
+		newName := args[0]
+		srcName := args[1]
+		orgShareName := ""
+		snapshot := ""
+		if fromSnapshot {
+			snapshot = srcName
+			shares, err := dsm.ShareList()
+			if err != nil {
+				fmt.Println(err)
+				os.Exit(1)
+			}
+
+			for _, share := range shares {
+				snaps, err := dsm.ShareSnapshotList(share.Name)
+				if err != nil {
+					fmt.Println(err)
+					os.Exit(1)
+				}
+				for _, snap := range snaps {
+					if snap.Time != srcName {
+						continue
+					}
+					orgShareName = share.Name
+					break
+				}
+				if orgShareName != "" {
+					break
+				}
+			}
+
+			if orgShareName == "" {
+				fmt.Println("Failed to find org Share of the snapshot")
+				os.Exit(1)
+			}
+		} else {
+			orgShareName = srcName
+		}
+		srcShare, err := dsm.ShareGet(orgShareName)
+		if err != nil {
+			fmt.Println(err)
+			os.Exit(1)
+		}
+
+		shareSpec := webapi.ShareCloneSpec{
+			Name: newName,
+			Snapshot: snapshot,
+			ShareInfo: webapi.ShareInfo{
+				Name:                newName,
+				VolPath:             srcShare.VolPath,
+				Desc:                "Cloned from "+srcName+" by synocli",
+				EnableRecycleBin:    srcShare.EnableRecycleBin,
+				RecycleBinAdminOnly: srcShare.RecycleBinAdminOnly,
+				NameOrg:             orgShareName,
+			},
+		}
+		fmt.Printf("newName: %s, fromSnapshot: %v (%s), orgShareName: %s\n", newName, fromSnapshot, snapshot, orgShareName)
+		_, err = dsm.ShareClone(shareSpec)
+		if err != nil {
+			fmt.Println(err)
+			os.Exit(1)
+		}
+
+		share, err := dsm.ShareGet(newName)
+		if err != nil {
+			fmt.Println(err)
+			os.Exit(1)
+		}
+
+		fmt.Printf("Success, ShareClone(%s, %s) new share = %#v\n", newName, srcName, share)
+	},
+}
+
+var cmdShareSnapshotCreate = &cobra.Command{
+	Use:   "snap_create <src name> <desc> <is_lock [true|false]>",
+	Short: "create share snapshot (only share located in btrfs volume can take snapshots)",
+	Args:  cobra.MinimumNArgs(3),
+	Run: func(cmd *cobra.Command, args []string) {
+		dsm, err := LoginDsmForTest()
+		if err != nil {
+			fmt.Println(err)
+			os.Exit(1)
+		}
+		defer func() {
+			dsm.Logout()
+		}()
+
+		spec := webapi.ShareSnapshotCreateSpec{
+			ShareName: args[0],
+			Desc:      args[1],
+			IsLocked:  utils.StringToBoolean(args[2]),
+		}
+
+		fmt.Printf("spec = %#v\n", spec)
+		snapTime, err := dsm.ShareSnapshotCreate(spec)
+		if err != nil {
+			fmt.Println(err)
+			os.Exit(1)
+		}
+		fmt.Printf("resp = %s\n", snapTime)
+
+		snaps, err := dsm.ShareSnapshotList(spec.ShareName)
+		if err != nil {
+			fmt.Println(err)
+			os.Exit(1)
+		}
+
+		var snapshot webapi.ShareSnapshotInfo
+		for _, snap := range snaps {
+			if snap.Time == snapTime {
+				snapshot = snap
+				break
+			}
+		}
+
+		fmt.Printf("Success, ShareSnapshotCreate(%s), snap = %#v\n", args[0], snapshot)
+	},
+}
+
+var cmdShareSnapshotDelete = &cobra.Command{
+	Use:   "snap_delete <share name> <snapshot time>",
+	Short: "delete share snapshot",
+	Args:  cobra.MinimumNArgs(2),
+	Run: func(cmd *cobra.Command, args []string) {
+		dsm, err := LoginDsmForTest()
+		if err != nil {
+			fmt.Println(err)
+			os.Exit(1)
+		}
+		defer func() {
+			dsm.Logout()
+		}()
+
+		if err := dsm.ShareSnapshotDelete(args[1], args[0]); err != nil {
+			fmt.Println(err)
+			os.Exit(1)
+		}
+		fmt.Printf("Success, ShareSnapshotDelete(%s, %s)\n", args[1], args[0])
+	},
+}
+
+func shareSnapshotListAll(dsm *webapi.DSM, shareName string) ([]webapi.ShareSnapshotInfo, error) {
+	if shareName != "" {
+		return dsm.ShareSnapshotList(shareName)
+	}
+
+	var infos []webapi.ShareSnapshotInfo
+	shares, err := dsm.ShareList()
+	if err != nil {
+		fmt.Println(err)
+		return nil, err
+	}
+
+	for _, share := range shares {
+		snaps, err := dsm.ShareSnapshotList(share.Name)
+		if err != nil {
+			fmt.Println(err)
+			return nil, err
+		}
+		infos = append(infos, snaps...)
+	}
+
+	return infos, nil
+}
+
+var cmdShareSnapshotList = &cobra.Command{
+	Use:   "snap_list [<share name>]",
+	Short: "list share snapshots",
+	Args:  cobra.MinimumNArgs(0),
+	Run: func(cmd *cobra.Command, args []string) {
+		dsm, err := LoginDsmForTest()
+		if err != nil {
+			fmt.Println(err)
+			os.Exit(1)
+		}
+		defer func() {
+			dsm.Logout()
+		}()
+
+		shareName := ""
+		if len(args) >= 1 {
+			shareName = args[0]
+		}
+
+		snaps, err := shareSnapshotListAll(dsm, shareName)
+		if err != nil {
+			fmt.Println(err)
+			os.Exit(1)
+		}
+
+		tw := tabwriter.NewWriter(os.Stdout, 8, 0, 2, ' ', 0)
+
+		fmt.Fprintf(tw, "id:\tTime:\tDesc:\tLock:\tScheduleSnapshot:\n")
+		for i, info := range snaps {
+			fmt.Fprintf(tw, "%d\t%-32s\t", i, info.Time)
+			fmt.Fprintf(tw, "%-32s\t", info.Desc)
+			fmt.Fprintf(tw, "%v\t", info.Lock)
+			fmt.Fprintf(tw, "%v\t", info.ScheduleSnapshot)
+			fmt.Fprintf(tw, "\n")
+
+			_ = tw.Flush()
+		}
+		fmt.Printf("Success, ShareSnapList(%s)\n", shareName)
+	},
+}
+
+func createSharePermission(name string, acl string) *webapi.SharePermission {
+	var permission webapi.SharePermission
+	permission.Name = name
+
+	switch strings.ToLower(acl) {
+	case "rw":
+		permission.IsWritable = true
+	case "ro":
+		permission.IsReadonly = true
+	case "no":
+		permission.IsDeny = true
+	default:
+		fmt.Println("[ERROR] Unknown ACL")
+		return nil
+	}
+	return &permission
+}
+
+func getShareLocalUserPermission(dsm *webapi.DSM, shareName string, userName string) (*webapi.SharePermission, error) {
+	infos, err := dsm.SharePermissionList(shareName, "local_user")
+	if err != nil {
+		return nil, err
+	}
+	for _, info := range infos {
+		if info.Name == userName {
+			return &info, nil
+		}
+	}
+	return nil, fmt.Errorf("Permission Not Found.")
+}
+
+var cmdSharePermissionList = &cobra.Command{
+	Use:   "permission_list <name> [local_user|local_group|system]",
+	Short: "list permissions",
+	Args:  cobra.MinimumNArgs(1),
+	Run: func(cmd *cobra.Command, args []string) {
+		dsm, err := LoginDsmForTest()
+		if err != nil {
+			fmt.Println(err)
+			os.Exit(1)
+		}
+		defer func() {
+			dsm.Logout()
+		}()
+
+		userGroupType := "local_user"
+		if len(args) >= 2 {
+			userGroupType = args[1]
+		}
+
+		infos, err := dsm.SharePermissionList(args[0], userGroupType)
+		if err != nil {
+			fmt.Println(err)
+			os.Exit(1)
+		}
+
+		tw := tabwriter.NewWriter(os.Stdout, 8, 0, 2, ' ', 0)
+
+		fmt.Fprintf(tw, "%s\t%-20s\t%-10s\t%-12s\t%-12s\t%-10s\t%-10s\n", "id:", "Name:", "IsAdmin", "IsReadonly:", "IsWritable:", "IsDeny:", "IsCustom:")
+		for i, info := range infos {
+			fmt.Fprintf(tw, "%d\t", i)
+			fmt.Fprintf(tw, "%-20s\t", info.Name)
+			fmt.Fprintf(tw, "%-10v\t", info.IsAdmin)
+			fmt.Fprintf(tw, "%-12v\t", info.IsReadonly)
+			fmt.Fprintf(tw, "%-12v\t", info.IsWritable)
+			fmt.Fprintf(tw, "%-10v\t", info.IsDeny)
+			fmt.Fprintf(tw, "%-10v\t", info.IsCustom)
+			fmt.Fprintf(tw, "\n")
+
+			_ = tw.Flush()
+		}
+
+		fmt.Printf("Success, SharePermissionList(%s)\n", args[0])
+	},
+}
+
+var cmdSharePermissionSet = &cobra.Command{
+	Use:   "permission_set <share name> <username> <[rw|ro|no]>",
+	Short: "set permission",
+	Args:  cobra.MinimumNArgs(3),
+	Run: func(cmd *cobra.Command, args []string) {
+		dsm, err := LoginDsmForTest()
+		if err != nil {
+			fmt.Println(err)
+			os.Exit(1)
+		}
+		defer func() {
+			dsm.Logout()
+		}()
+
+		shareName := args[0]
+		userName := args[1]
+		userGroupType := "local_user"
+		permission := createSharePermission(userName, args[2])
+		if permission == nil {
+			fmt.Println("Failed. Invalid Argument.")
+			os.Exit(1)
+		}
+		permissions := append([]*webapi.SharePermission{}, permission)
+
+		spec := webapi.SharePermissionSetSpec{
+			Name: shareName,
+			UserGroupType: userGroupType,
+			Permissions: permissions,
+		}
+
+		fmt.Printf("spec = %#v\n", spec)
+		if err := dsm.SharePermissionSet(spec); err != nil {
+			fmt.Println(err)
+			os.Exit(1)
+		}
+		newPermission, err := getShareLocalUserPermission(dsm, shareName, userName)
+		if err != nil {
+			fmt.Printf("Failed to get share local_user permission(%s, %s): %v\n", shareName, userName, err)
+			os.Exit(1)
+		}
+
+		fmt.Printf("Success, SharePermissionSet(%s), newPermission: %#v\n", shareName, newPermission)
+	},
+}
+
+var cmdShareSet = &cobra.Command{
+	Use:   "set <share name> <new size>",
+	Short: "share set (only share located in btrfs volume can be resized)",
+	Args:  cobra.MinimumNArgs(2),
+	Run: func(cmd *cobra.Command, args []string) {
+		dsm, err := LoginDsmForTest()
+		if err != nil {
+			fmt.Println(err)
+			os.Exit(1)
+		}
+		defer func() {
+			dsm.Logout()
+		}()
+
+		shareName := args[0]
+		newSize, err := strconv.ParseInt(args[1], 10, 64)
+		if err != nil {
+		    fmt.Println(err)
+			os.Exit(1)
+		}
+
+		share, err := dsm.ShareGet(shareName)
+		if err != nil {
+			fmt.Println(err)
+			os.Exit(1)
+		}
+		fmt.Printf("old = %#v\n", share)
+
+		newSizeInMB := utils.BytesToMBCeil(newSize)
+		updateInfo := webapi.ShareUpdateInfo{
+			Name:           share.Name,
+			VolPath:        share.VolPath,
+			QuotaForCreate: &newSizeInMB,
+		}
+		if err := dsm.ShareSet(shareName, updateInfo); err != nil {
+			fmt.Println(err)
+			os.Exit(1)
+		}
+
+		newShare, err := dsm.ShareGet(shareName)
+		if err != nil {
+			fmt.Println(err)
+			os.Exit(1)
+		}
+		fmt.Printf("new = %#v\n", newShare)
+
+		fmt.Printf("Success, ShareSet(%s), quota: %v -> %v MB\n", shareName, share.QuotaValueInMB, newShare.QuotaValueInMB)
+	},
+}
+
+
+func init() {
+	cmdShare.AddCommand(cmdShareGet)
+	cmdShare.AddCommand(cmdShareCreate)
+	cmdShare.AddCommand(cmdShareDelete)
+	cmdShare.AddCommand(cmdShareList)
+	cmdShare.AddCommand(cmdShareClone)
+	cmdShare.AddCommand(cmdShareSnapshotCreate)
+	cmdShare.AddCommand(cmdShareSnapshotDelete)
+	cmdShare.AddCommand(cmdShareSnapshotList)
+
+	cmdShare.AddCommand(cmdSharePermissionList)
+	cmdShare.AddCommand(cmdSharePermissionSet)
+	cmdShare.AddCommand(cmdShareSet)
+}

test/sanity/sanity-test-secret-file-template.yaml

@@ -0,0 +1,12 @@
+# CreateVolumeSecret:
+# DeleteVolumeSecret:
+# ControllerPublishVolumeSecret:
+# ControllerUnpublishVolumeSecret:
+# ControllerValidateVolumeCapabilitiesSecret:
+NodeStageVolumeSecret:
+  username: "username"
+  password: "password"
+# NodePublishVolumeSecret:
+# CreateSnapshotSecret:
+# DeleteSnapshotSecret:
+# ControllerExpandVolumeSecret:

test/sanity/sanity_test.go

@@ -15,6 +15,7 @@ import (
 
 const (
 	ConfigPath = "./../../config/client-info.yml"
+	SecretsFilePath = "./sanity-test-secret-file.yaml"
 )
 
 func TestSanity(t *testing.T) {
@@ -69,6 +70,15 @@ func TestSanity(t *testing.T) {
 	testConfig.TargetPath = targetPath
 	testConfig.StagingPath = stagingPath
 	testConfig.Address = endpoint
+	testConfig.SecretsFile = SecretsFilePath
+
+	// Set Input parameters for test
+	testConfig.TestVolumeParameters = map[string]string{
+		"protocol": "smb",
+	}
+
+	// testConfig.TestVolumeAccessType = "block" // raw block
+
 	// Run test
 	sanity.Test(t, testConfig)
 }