# Confirm Centos kernel version
uname -a
# e.g. 3.10.0-693.el7.x86_64

# use google search keywords [centos7.4 3.10.0-693.el7.x86_64] and the first result may be the right download link. Download kernel-devel package and run rpm install command.

# wget https://buildlogs.centos.org/c7.1708.00/kernel/20170822030048/3.10.0-693.el7.x86_64/kernel-devel-3.10.0-693.el7.x86_64.rpm

# rpm -ivh kernel-devel-3.10.0-693.el7.x86_64.rpm

blacklist nouveau
blacklist lbm-nouveau
options nouveau modeset=0
alias nouveau off
alias lbm-nouveau off

echo options nouveau modeset=0 | sudo tee -a /etc/modprobe.d/nouveau-kms.conf

yum provides */lspci
yum install pciutils

# vim $install_package_path/diamond/config.js

# Installation of gpu driver is custom by default that means  the GPU Driver on the hosts is Pre-installed.
If you want to install gpu driver by the diamond,you can set the value of the nvidiaDriverVersion field.

############# run separately ############
##### Please ensure use ubuntu user. #####
# make sure install_package_path is install package path.
sudo su - ubuntu
cd $install_package_path

# Create tmux session for installation
# $ tmux new -s senselink
./setup_diamond.sh |& tee -a ./setup_diamond.log

workflow installation has completed

     ***************
   *******************
     ***************
       ***********
         *******
           ***
            *

Diamond successfully installed

# find parameter service-node-port-range and change value to 1-65535
sudo vim /etc/kubernetes/manifests/kube-apiserver.yaml
sudo systemctl daemon-reload
sudo systemctl restart kubelet

# add diamond.key to host private key list
ssh-agent bash
ssh-add /data/diamond/shells/diamond.key

##############################################
# Please ensure existing id_rsa wasn't used before run below command.
##############################################
# copy diamond.key as host private key
cp -i /data/diamond/shells/diamond.key /home/ubuntu/.ssh/id_rsa

host_ip=10.10.10.x
scp /data/diamond/report/k8sKubeConfig $host_ip:~/kubeconfig
ssh $host_ip 'mkdir -p ~/.kube/ && mv ~/kubeconfig ~/.kube/config && sudo mkdir -p /root/.kube/ && sudo cp ~/.kube/config /root/.kube/'

# adding frequent used commands
echo '
alias kaf="kubectl apply -f"
alias kdf="kubectl delete -f"
alias kg="kubectl get"
alias kga="kubectl get all"
alias kgp="kubectl get pods"
alias kd="kubectl describe"
alias kdp="kubectl describe pods"
alias kl="kubectl logs"
alias keti="kubectl exec -ti"
alias sk="kubectl -n senselink"
alias skg="kubectl -n senselink get"
alias skga="kubectl -n senselink get all"
alias skgp="kubectl -n senselink get pods"
alias skd="kubectl -n senselink describe"
alias skdp="kubectl -n senselink describe pods"
alias skl="kubectl -n senselink logs"
alias sketi="kubectl -n senselink exec -ti"
alias ck="kubectl -n component"
alias ckg="kubectl -n component get"
alias ckga="kubectl -n component get all"
alias ckgp="kubectl -n component get pods"
alias ckd="kubectl -n component describe"
alias ckdp="kubectl -n component describe pods"
alias ckl="kubectl -n component logs"
alias cketi="kubectl -n component exec -ti"
' >> ~/.bashrc
source ~/.bashrc

# make sure that which node  has pods with ImageInspectError
kubectl get pod -o wide

# delete image in tht pod with ImageInspectError
docker rmi -f imageId 
docker system prune 

# delete the pod with ImageInspectError 
kubectl delete pod

# get DUID of all Link node
kubectl logs -f feature-0 -n senselink | grep DUID
kubectl logs -f feature-1 -n senselink | grep DUID
kubectl logs -f feature-2 -n senselink | grep DUID

# delete Feature pods
kubectl delete -f  $install_package_path/senselink/multi-done/statefulset/feature-statefulset.yaml

# delete Feature pvc
kubectl delete pvc -n senselink feature-claim-feature-0
kubectl delete pvc -n senselink feature-claim-feature-1
kubectl delete pvc -n senselink feature-claim-feature-2

# vim $install_package_path/senselink/multi-done/statefulset/feature-statefulset.yaml  
apiVersion: apps/v1
kind: StatefulSet
metadata:
  creationTimestamp: null
  labels:
    senselink: feature
  name: feature
  namespace: senselink
spec:
  replicas: 1
  serviceName: feature
  selector:
    matchLabels:
      senselink: feature
  template:
    metadata:
      creationTimestamp: null
      labels:
        senselink: feature
    spec:
      containers:
      - image: registry.sensetime.com/jcv/feature_service:v2.3.0
        name: feature
        ports:
        - containerPort: 50052
        resources:
          limits:
            memory: "1073741824"
        stdin: true
        tty: true
        volumeMounts:
        - mountPath: /go/log
          name: feature-claim
        - mountPath: /go/conf/qa.toml
          subPath: qa.toml
          name: feature-qa
        - mountPath: /go/conf/feature.toml
          subPath: feature.toml
          name: feature-conf
        - mountPath: /go/license/
          name: feature-sub-license
      restartPolicy: Always
      volumes:
      - name: feature-claim
        persistentVolumeClaim:
          claimName: feature-claim
      - name: feature-qa
        configMap:
          name: feature-qa
      - name: feature-conf
        configMap:
          name: feature-conf
      - name: feature-sub-license
        hostPath:
          path: /etc/senselink/license
      nodeSelector:
        app: senselink
        feature: enable
  volumeClaimTemplates:
  - metadata:
      name: feature-claim
    spec:
      accessModes:
      - ReadWriteOnce
      resources:
        requests:
          storage: 1Gi

# show all k8s nodes with labels
kubectl get node  --show-labels

# Only label the node you want to activate this time.
kubectl label node sz-office2-172-20-52-184-172.20.52.184 feature=enable --overwrite

# Label other node to diable feature service in this node.
kubectl label node sz-office2-172-20-52-184-172.20.52.184 feature=disable --overwrite

# delete the redis key `authorize-license-status` or wait 5 minutes.
redis-cli -h 10.10.41.16  -a 2CH189BxbsKceg4l
10.10.41.16:0>del authorize-license-status

kubectl apply -f $install_package_path/senselink/multi-done/statefulset/feature-statefulset.yaml

# vim $install_package_path/senselink/multi-done/statefulset/feature-statefulset.yaml  
apiVersion: apps/v1
kind: StatefulSet
metadata:
  creationTimestamp: null
  labels:
    senselink: feature
  name: feature
  namespace: senselink
spec:
  replicas: 3  #number of line nodes
  serviceName: feature
  selector:
    matchLabels:
      senselink: feature
  template:
    metadata:
      creationTimestamp: null
      labels:
        senselink: feature
    spec:
      containers:
      - image: registry.sensetime.com/jcv/feature_service:v2.3.0
        name: feature
        ports:
        - containerPort: 50052
        resources:
          limits:
            memory: "1073741824"
        stdin: true
        tty: true
        volumeMounts:
        - mountPath: /go/log
          name: feature-claim
        - mountPath: /go/conf/qa.toml
          subPath: qa.toml
          name: feature-qa
        - mountPath: /go/conf/feature.toml
          subPath: feature.toml
          name: feature-conf
        - mountPath: /go/license/
          name: feature-sub-license
      restartPolicy: Always
      volumes:
      - name: feature-claim
        persistentVolumeClaim:
          claimName: feature-claim
      - name: feature-qa
        configMap:
          name: feature-qa
      - name: feature-conf
        configMap:
          name: feature-conf
      - name: feature-sub-license
        hostPath:
          path: /etc/senselink/license
      nodeSelector:
        app: senselink
  volumeClaimTemplates:
  - metadata:
      name: feature-claim
    spec:
      accessModes:
      - ReadWriteOnce
      resources:
        requests:
          storage: 1Gi

kubectl delete -f  $install_package_path/senselink/multi-done/statefulset/feature-statefulset.yaml

# delete Feature pvc
kubectl delete pvc -n senselink feature-claim-feature-0
kubectl delete pvc -n senselink feature-claim-feature-1
kubectl delete pvc -n senselink feature-claim-feature-2 

kubectl apply -f  $install_package_path/senselink/multi-done/statefulset/feature-statefulset.yaml

# vim $install_package_path/senselink/multi-done/configmap/web-configmap.yaml
# 100.122.0.0/16 is the Subnet mask of external SLB Ip
...
location /sl {
      add_header 'Access-Control-Allow-Origin' *;
      proxy_pass $upstream_apiv2;
      proxy_set_header Host $host;
      proxy_set_header X-Real-IP $remote_addr;
      proxy_set_header X-Real-PORT $remote_port;
      proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
      proxy_set_header Cookie $http_cookie;
      proxy_set_header X-Forwarded-Proto $scheme;
      proxy_redirect off;
      set_real_ip_from 100.122.0.0/16;
      real_ip_recursive on;
      real_ip_header  X-Forwarded-For;
    }
...

vim $install_package_path/senselink/multi-done/configmap/apiv2-configmap.yaml

########
keeper:
  token:
    expires: 43200  # Token validity period in seconds

kubectl apply -f $install_package_path/senselink/multi-done/configmap/apiv2-configmap.yaml
kubectl delete -f $install_package_path/senselink/multi-done/statefulset/apiv2-statefulset.yaml
kubectl apply -f $install_package_path/senselink/multi-done/statefulset/apiv2-statefulset.yaml

$ kubectl -n senselink get pod

NAME                         READY     STATUS             RESTARTS   AGE
apiv2-0                      0/1       CrashLoopBackOff   37         172m
apiv2-1                      0/1       CrashLoopBackOff   37         172m
apiv2-2                      0/1       CrashLoopBackOff   36         172m
emq-5c5b6fbbf9-wmxw9         1/1       Running            0          3h9m
feature-0                    1/1       Running            0          3h9m
feature-1                    1/1       Running            0          3h8m
feature-2                    1/1       Running            0          3h8m
mmo-init-job-nbm59           0/1       Completed          0          3h10m
...

$ kubectl -n senselink logs apiv2-0
...
Caused by: org.springframework.beans.factory.UnsatisfiedDependencyException: Error creating bean with name 'InfraService': Unsatisfied dependency expressed through field 'port'; nested exception is org.springframework.beans.TypeMismatchException: Failed to convert value of type 'java.lang.String' to required type 'java.lang.Integer'; nested exception is java.lang.NumberFormatException: For input string: "tcp://10.108.12.226:50009"
...

# delete infra-service fs-service and apiv2 pods
$ kubectl delete -f /data/senselink/multi-done/service/infra-service.yaml
service "infra" deleted
$ kubectl delete -f /data/senselink/multi-done/service/fs-service.yaml
service "fs" deleted
$ kubectl delete -f /data/senselink/multi-done/statefulset/apiv2-statefulset.yaml
statefulset.apps "apiv2" deleted

# re-deploy apiv2 pods
$ kubectl apply -f /data/senselink/multi-done/statefulset/apiv2-statefulset.yaml
statefulset.apps/apiv2 created
# chech all apiv2 pods are running
$ kubectl -n senselink get pods
NAME                         READY     STATUS      RESTARTS   AGE
apiv2-0                      1/1       Running     0          4m15s
apiv2-1                      1/1       Running     0          4m5s
apiv2-2                      1/1       Running     0          3m59s
...

# redeploy infra-service
$ kubectl apply -f /data/senselink/multi-done/service/infra-service.yaml
service/infra created

# redeploy fs-service
$ kubectl apply -f /data/senselink/multi-done/service/fs-service.yaml
service/fs created

docker: Error response from daemon: failed to start shim: exec: "docker: Error response from daemon: failed to start shim: exec: "docker-containerd-shim": executable file not found in $PATH: unknown.
": executable file not found in $PATH: unknown.

docker: Error response from daemon: OCI runtime create failed: unable to retrieve OCI runtime error (open /run/docker/containerd/daemon/io.containerd.runtime.v1.linux/moby/b123c36ddea60f7bf562946b7ffb9520023fcb4946b3e7f9941a564a46bad28c/log.json: no such file or directory): exec: "docker-runc": executable file not found in $PATH: unknown.

# Disable apt package auto upgrade service
sudo systemctl stop apt-daily.service apt-daily.timer apt-daily-upgrade.timer apt-daily-upgrade.service
sudo systemctl disable apt-daily.service apt-daily.timer apt-daily-upgrade.timer apt-daily-upgrade.service

[ubuntu@saas-i-prod-saasdiamondali10 ~]$ kubectl get pods
No resources found.

[ubuntu@saas-i-prod-saasdiamondali10 ~]$ kubectl get pods
No resources found.

[ubuntu@saas-i-prod-saasdiamondali10 ~]$ kubectl get pods

server: https://10.0.222.212:9443

server: https://10.0.222.222:6443

[ubuntu@saas-i-prod-saasdiamondali10 ~]$ kubectl get pods 
No resources found.

$ sudo docker ps -a | grep haproxy && sudo docker ps -a | grep keepalive 
646f3c72799c registry.sensetime.com/diamond/haproxy:2.1.2-1 "/docker-entrypoint.…" 6 weeks ago Exited (0) 21 hours ago diamond-haproxy 
5ca5eeb00609 registry.sensetime.com/diamond/keepalived:2.0.19 "/container/tool/run" 6 weeks ago Exited (0) 21 hours ago diamond-keepalived

$ sudo docker start 646f3c72799c && sudo docker start 5ca5eeb00609

[ubuntu@saas-i-prod-saasdiamondali10 ~]$ kubectl get pods 
No resources found.