k8s 配置hadoop集群，nfs作为存储

目录

一、简介

二、nfs服务&nfs-provisioner配置

1、k8S服务器需安装nfs客户端

2、nfs服务端安装配置

3、使用nfs-provisioner动态创建PV  （文件已修改）

三、hadoop配置文件

1、# cat hadoop.yaml

2、# cat hadoop-datanode.yaml

3、# cat yarn-node.yaml 

4、执行文件并查看

5、联通性验证

 四、报错&解决

1、nfs报错 

 2、nfs报错 

一、简介

基础环境使用kubeasz（https://github.com/easzlab/kubeasz）配置K8S环境.
K8S配置示例:https://blog.csdn.net/zhangxueleishamo/article/details/108670578
使用nfs作为资源存储。

二、nfs服务&nfs-provisioner配置

1、k8S服务器需安装nfs客户端

yum -y install nfs-utils

2、nfs服务端安装配置

yum -y install nfs-utils rpcbind nfs-server

# cat /etc/exports
/data/hadoop	*(rw,no_root_squash,no_all_squash,sync)
###权限及目录配置，具体不再说明

systemctl start rpcbind 
systemctl enable rpcbind

systemctl start nfs
systemctl enable nfs

3、使用nfs-provisioner动态创建PV  （文件已修改）

# cat nfs-provisioner.yaml 

apiVersion: v1
kind: ServiceAccount
metadata:
  name: nfs-client-provisioner
  namespace: dev
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: nfs-client-provisioner-runner
rules:
  - apiGroups: [""]
    resources: ["persistentvolumes"]
    verbs: ["get", "list", "watch", "create", "delete"]
  - apiGroups: [""]
    resources: ["persistentvolumeclaims"]
    verbs: ["get", "list", "watch", "update"]
  - apiGroups: ["storage.k8s.io"]
    resources: ["storageclasses"]
    verbs: ["get", "list", "watch"]
  - apiGroups: [""]
    resources: ["events"]
    verbs: ["list", "watch", "create", "update", "patch"]
  - apiGroups: [""]
    resources: ["services"]
    verbs: ["get"]
  - apiGroups: ["extensions"]
    resources: ["podsecuritypolicies"]
    resourceNames: ["nfs-provisioner"]
    verbs: ["use"]
  - apiGroups: [""]
    resources: ["endpoints"]
    verbs: ["get", "list", "watch", "create", "update", "patch"]
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: run-nfs-client-provisioner
subjects:
  - kind: ServiceAccount
    name: nfs-client-provisioner
    namespace: dev 
roleRef:
  kind: ClusterRole
  name: nfs-client-provisioner-runner
  apiGroup: rbac.authorization.k8s.io

---
kind: Deployment
apiVersion: apps/v1
metadata:
  name: nfs-client-provisioner
  namespace: dev
spec:
  replicas: 1
  strategy:
    type: Recreate
  selector:
    matchLabels:
      app: nfs-client-provisioner
  template:
    metadata:
      labels:
        app: nfs-client-provisioner
    spec:
      serviceAccountName: nfs-client-provisioner
      containers:
        - name: nfs-client-provisioner
          #image: quay.io/external_storage/nfs-client-provisioner:latest
          image: jmgao1983/nfs-client-provisioner:latest
          imagePullPolicy: IfNotPresent
          volumeMounts:
            - name: nfs-client-root
              mountPath: /persistentvolumes
          env:
            - name: PROVISIONER_NAME
              # 此处供应者名字供storageclass调用
              value: nfs-storage
            - name: NFS_SERVER
              value: 10.2.1.190
            - name: NFS_PATH
              value: /data/hadoop
      volumes:
        - name: nfs-client-root
          nfs:
            server: 10.2.1.190
            path: /data/hadoop

---
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
  name: nfs-storage
  annotations:
    storageclass.kubernetes.io/is-default-class: "true"
provisioner: nfs-storage
volumeBindingMode: Immediate
reclaimPolicy: Delete


###执行并查看sa & sc ##

# kubectl apply -f nfs-provisioner.yaml 

# kubectl get sa,sc  -n dev 
NAME                                    SECRETS   AGE
serviceaccount/nfs-client-provisioner   1         47m

NAME                                                PROVISIONER   RECLAIMPOLICY   VOLUMEBINDINGMODE   ALLOWVOLUMEEXPANSION   AGE
storageclass.storage.k8s.io/nfs-storage (default)   nfs-storage   Delete          Immediate           false                  45m

三、hadoop配置文件

1、# cat hadoop.yaml

apiVersion: v1
kind: ConfigMap
metadata:
  name: kube-hadoop-conf
  namespace: dev
data:
  HDFS_MASTER_SERVICE: hadoop-hdfs-master
  HDOOP_YARN_MASTER: hadoop-yarn-master
---
apiVersion: v1
kind: Service
metadata:
  name: hadoop-hdfs-master
  namespace: dev
spec:
  type: NodePort
  selector:
    name: hdfs-master
  ports:
    - name: rpc
      port: 9000
      targetPort: 9000
    - name: http
      port: 50070
      targetPort: 50070
      nodePort: 32007
---
apiVersion: v1
kind: Service
metadata:
  name: hadoop-yarn-master
  namespace: dev
spec:
  type: NodePort
  selector:
    name: yarn-master
  ports:
     - name: "8030"
       port: 8030
     - name: "8031"
       port: 8031
     - name: "8032"
       port: 8032
     - name: http
       port: 8088
       targetPort: 8088
       nodePort: 32088
---
apiVersion: v1
kind: Service
metadata:
  name: yarn-node
  namespace: dev
spec:
  clusterIP: None
  selector:
    name: yarn-node
  ports:
     - port: 8040

2、# cat hadoop-datanode.yaml

apiVersion: apps/v1
kind: Deployment
metadata:
  name: hdfs-master
  namespace: dev
spec:
  replicas: 1
  selector:
    matchLabels:
      name: hdfs-master
  template:
    metadata:
      labels:
        name: hdfs-master
    spec:
      containers:
        - name: hdfs-master
          image: kubeguide/hadoop:latest
          imagePullPolicy: IfNotPresent
          ports:
            - containerPort: 9000
            - containerPort: 50070
          env:
            - name: HADOOP_NODE_TYPE
              value: namenode
            - name: HDFS_MASTER_SERVICE
              valueFrom:
                configMapKeyRef:
                  name: kube-hadoop-conf
                  key: HDFS_MASTER_SERVICE
            - name: HDOOP_YARN_MASTER
              valueFrom:
                configMapKeyRef:
                  name: kube-hadoop-conf
                  key: HDOOP_YARN_MASTER
      restartPolicy: Always
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: hadoop-datanode
  namespace: dev
spec:
  replicas: 3
  selector:
    matchLabels:
      name: hadoop-datanode
  serviceName: hadoop-datanode
  template:
    metadata:
      labels:
        name: hadoop-datanode
    spec:
      containers:
        - name: hadoop-datanode
          image: kubeguide/hadoop:latest
          imagePullPolicy: IfNotPresent
          ports:
            - containerPort: 9000
            - containerPort: 50070
          volumeMounts:
            - name: data
              mountPath: /root/hdfs/
              subPath: hdfs
            - name: data
              mountPath: /usr/local/hadoop/logs/
              subPath: logs
          env:
            - name: HADOOP_NODE_TYPE
              value: datanode
            - name: HDFS_MASTER_SERVICE
              valueFrom:
                configMapKeyRef:
                  name: kube-hadoop-conf
                  key: HDFS_MASTER_SERVICE
            - name: HDOOP_YARN_MASTER
              valueFrom:
                configMapKeyRef:
                  name: kube-hadoop-conf
                  key: HDOOP_YARN_MASTER
      restartPolicy: Always
  volumeClaimTemplates:
    - metadata:
        name: data
        namespace: dev
      spec:
        accessModes:
          - ReadWriteMany
        resources:
          requests:
            storage: 2Gi
        storageClassName: "nfs-storage"

3、# cat yarn-node.yaml 

apiVersion: apps/v1
kind: Deployment
metadata:
  name: yarn-master
  namespace: dev
spec:
  replicas: 1
  selector:
    matchLabels:
      name: yarn-master
  template:
    metadata:
      labels:
        name: yarn-master
    spec:
      containers:
        - name: yarn-master
          image: kubeguide/hadoop:latest
          imagePullPolicy: IfNotPresent
          ports:
            - containerPort: 9000
            - containerPort: 50070
          env:
            - name: HADOOP_NODE_TYPE
              value: resourceman
            - name: HDFS_MASTER_SERVICE
              valueFrom:
                configMapKeyRef:
                  name: kube-hadoop-conf
                  key: HDFS_MASTER_SERVICE
            - name: HDOOP_YARN_MASTER
              valueFrom:
                configMapKeyRef:
                  name: kube-hadoop-conf
                  key: HDOOP_YARN_MASTER
      restartPolicy: Always
---
 
apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: yarn-node
  namespace: dev
spec:
  replicas: 3
  selector:
    matchLabels:
      name: yarn-node
  serviceName: yarn-node
  template:
    metadata:
      labels:
        name: yarn-node
    spec:
      containers:
        - name: yarn-node
          image: kubeguide/hadoop:latest
          imagePullPolicy: IfNotPresent
          ports:
            - containerPort: 8040
            - containerPort: 8041
            - containerPort: 8042
          volumeMounts:
            - name: yarn-data
              mountPath: /root/hdfs/
              subPath: hdfs
            - name: yarn-data
              mountPath: /usr/local/hadoop/logs/
              subPath: logs
          env:
            - name: HADOOP_NODE_TYPE
              value: yarnnode
            - name: HDFS_MASTER_SERVICE
              valueFrom:
                configMapKeyRef:
                  name: kube-hadoop-conf
                  key: HDFS_MASTER_SERVICE
            - name: HDOOP_YARN_MASTER
              valueFrom:
                configMapKeyRef:
                  name: kube-hadoop-conf
                  key: HDOOP_YARN_MASTER
      restartPolicy: Always
  volumeClaimTemplates:
    - metadata:
        name: yarn-data
        namespace: dev
      spec:
        accessModes:
          - ReadWriteMany
        resources:
          requests:
            storage: 2Gi
        storageClassName: "nfs-storage"

4、执行文件并查看

kubectl apply -f hadoop.yaml
kubectl apply -f hadoop-datanode.yaml
kubectl apply -f yarn-node.yaml


# kubectl get pv,pvc -n dev 
NAME                                                        CAPACITY   ACCESS MODES   RECLAIM POLICY   STATUS   CLAIM                        STORAGECLASS   REASON   AGE
persistentvolume/pvc-2bf83ccf-85eb-43d7-8d49-10a2617c1bde   2Gi        RWX            Delete           Bound    dev/data-hadoop-datanode-0   nfs-storage             34m
persistentvolume/pvc-5ecff2b2-ea9d-4d6f-851b-0ab2cecbbe54   2Gi        RWX            Delete           Bound    dev/yarn-data-yarn-node-1    nfs-storage             32m
persistentvolume/pvc-91132f6d-a3e1-4938-b8d7-674d6b0656a8   2Gi        RWX            Delete           Bound    dev/data-hadoop-datanode-2   nfs-storage             34m
persistentvolume/pvc-a44adf12-2505-4133-ab57-99a61c4d4476   2Gi        RWX            Delete           Bound    dev/data-hadoop-datanode-1   nfs-storage             34m
persistentvolume/pvc-c4bf1e26-936f-46f6-8529-98d2699a916e   2Gi        RWX            Delete           Bound    dev/yarn-data-yarn-node-2    nfs-storage             32m
persistentvolume/pvc-e6d360be-2f72-4c47-a99b-fee79ca5e03b   2Gi        RWX            Delete           Bound    dev/yarn-data-yarn-node-0    nfs-storage             32m

NAME                                           STATUS   VOLUME                                     CAPACITY   ACCESS MODES   STORAGECLASS   AGE
persistentvolumeclaim/data-hadoop-datanode-0   Bound    pvc-2bf83ccf-85eb-43d7-8d49-10a2617c1bde   2Gi        RWX            nfs-storage    39m
persistentvolumeclaim/data-hadoop-datanode-1   Bound    pvc-a44adf12-2505-4133-ab57-99a61c4d4476   2Gi        RWX            nfs-storage    34m
persistentvolumeclaim/data-hadoop-datanode-2   Bound    pvc-91132f6d-a3e1-4938-b8d7-674d6b0656a8   2Gi        RWX            nfs-storage    34m
persistentvolumeclaim/yarn-data-yarn-node-0    Bound    pvc-e6d360be-2f72-4c47-a99b-fee79ca5e03b   2Gi        RWX            nfs-storage    32m
persistentvolumeclaim/yarn-data-yarn-node-1    Bound    pvc-5ecff2b2-ea9d-4d6f-851b-0ab2cecbbe54   2Gi        RWX            nfs-storage    32m
persistentvolumeclaim/yarn-data-yarn-node-2    Bound    pvc-c4bf1e26-936f-46f6-8529-98d2699a916e   2Gi        RWX            nfs-storage    32m


# kubectl get all -n dev  -o wide 
NAME                                         READY   STATUS    RESTARTS   AGE   IP            NODE         NOMINATED NODE   READINESS GATES
pod/hadoop-datanode-0                        1/1     Running   0          40m   172.20.4.65   10.2.1.194   <none>           <none>
pod/hadoop-datanode-1                        1/1     Running   0          35m   172.20.4.66   10.2.1.194   <none>           <none>
pod/hadoop-datanode-2                        1/1     Running   0          35m   172.20.4.67   10.2.1.194   <none>           <none>
pod/hdfs-master-5946bb8ff4-lt5mp             1/1     Running   0          40m   172.20.4.64   10.2.1.194   <none>           <none>
pod/nfs-client-provisioner-8ccc8b867-ndssr   1/1     Running   0          52m   172.20.4.63   10.2.1.194   <none>           <none>
pod/yarn-master-559c766d4c-jzz4s             1/1     Running   0          33m   172.20.4.68   10.2.1.194   <none>           <none>
pod/yarn-node-0                              1/1     Running   0          33m   172.20.4.69   10.2.1.194   <none>           <none>
pod/yarn-node-1                              1/1     Running   0          33m   172.20.4.70   10.2.1.194   <none>           <none>
pod/yarn-node-2                              1/1     Running   0          33m   172.20.4.71   10.2.1.194   <none>           <none>

NAME                         TYPE        CLUSTER-IP      EXTERNAL-IP   PORT(S)                                                       AGE   SELECTOR
service/hadoop-hdfs-master   NodePort    10.68.193.79    <none>        9000:26007/TCP,50070:32007/TCP                                40m   name=hdfs-master
service/hadoop-yarn-master   NodePort    10.68.243.133   <none>        8030:34657/TCP,8031:35352/TCP,8032:33633/TCP,8088:32088/TCP   40m   name=yarn-master
service/yarn-node            ClusterIP   None            <none>        8040/TCP                                                      40m   name=yarn-node

NAME                                     READY   UP-TO-DATE   AVAILABLE   AGE   CONTAINERS               IMAGES                                    SELECTOR
deployment.apps/hdfs-master              1/1     1            1           40m   hdfs-master              kubeguide/hadoop:latest                   name=hdfs-master
deployment.apps/nfs-client-provisioner   1/1     1            1           52m   nfs-client-provisioner   jmgao1983/nfs-client-provisioner:latest   app=nfs-client-provisioner
deployment.apps/yarn-master              1/1     1            1           33m   yarn-master              kubeguide/hadoop:latest                   name=yarn-master

NAME                                               DESIRED   CURRENT   READY   AGE   CONTAINERS               IMAGES                                    SELECTOR
replicaset.apps/hdfs-master-5946bb8ff4             1         1         1       40m   hdfs-master              kubeguide/hadoop:latest                   name=hdfs-master,pod-template-hash=5946bb8ff4
replicaset.apps/nfs-client-provisioner-8ccc8b867   1         1         1       52m   nfs-client-provisioner   jmgao1983/nfs-client-provisioner:latest   app=nfs-client-provisioner,pod-template-hash=8ccc8b867
replicaset.apps/yarn-master-559c766d4c             1         1         1       33m   yarn-master              kubeguide/hadoop:latest                   name=yarn-master,pod-template-hash=559c766d4c

NAME                               READY   AGE   CONTAINERS        IMAGES
statefulset.apps/hadoop-datanode   3/3     40m   hadoop-datanode   kubeguide/hadoop:latest
statefulset.apps/yarn-node         3/3     33m   yarn-node         kubeguide/hadoop:latest

访问http://ip:32007 和http://ip:32088 即可看到hadoop管理界面

5、联通性验证

在hdfs上创建一个目录

# kubectl exec -it hdfs-master-5946bb8ff4-lt5mp -n dev /bin/bash

# hdfs dfs -mkdir /BigData

在Hadoop WebUI界面查看刚才创建的目录

 四、报错&解决

1、nfs报错 

Unexpected error getting claim reference to claim "dev/data-hadoop-datanode-0": selfLink was empty, can't make reference

 问题原因： 没有权限创建

解决：

1、chmod 777 /data/hadoop #配置nfs共享文件权限，方便测试就777了

2、修改nfs-provisioner.yaml的rules

rules:
  - apiGroups: [""]
    resources: ["persistentvolumes"]
    verbs: ["get", "list", "watch", "create", "delete"]
  - apiGroups: [""]
    resources: ["persistentvolumeclaims"]
    verbs: ["get", "list", "watch", "update"]
  - apiGroups: ["storage.k8s.io"]
    resources: ["storageclasses"]
    verbs: ["get", "list", "watch"]
  - apiGroups: [""]
    resources: ["events"]
    verbs: ["list", "watch", "create", "update", "patch"]
  - apiGroups: [""]
    resources: ["services"]
    verbs: ["get"]
  - apiGroups: ["extensions"]
    resources: ["podsecuritypolicies"]
    resourceNames: ["nfs-provisioner"]
    verbs: ["use"]
  - apiGroups: [""]
    resources: ["endpoints"]
    verbs: ["get", "list", "watch", "create", "update", "patch"]

 2、nfs报错 

persistentvolume-controller  waiting for a volume to be created, either by external provisioner "nfs-provisioner" or manually created by system  administrator

 问题原因：selfLink导致，因为kubernetes 1.20版本 禁用了 selfLink导致

https://github.com/kubernetes/kubernetes/pull/94397

问题解决：

1、在hadoop的3个文件里 添加namespace与nfs的统一

2、修改kube-apiserver.yaml配置文件，添加如下内容

apiVersion: v1
-----
    - --tls-private-key-file=/etc/kubernetes/pki/apiserver.key
    - --feature-gates=RemoveSelfLink=false # 添加这个配置

本k8s使用kubeasz配置未找到该文件，需要直接修改所有的master的kube-apiserver服务的配置文件

# cat /etc/systemd/system/kube-apiserver.service
[Unit]
Description=Kubernetes API Server
Documentation=https://github.com/GoogleCloudPlatform/kubernetes
After=network.target

[Service]
ExecStart=/opt/kube/bin/kube-apiserver \
  --advertise-address=10.2.1.190 \
  --allow-privileged=true \
  --anonymous-auth=false \
  --api-audiences=api,istio-ca \
  --authorization-mode=Node,RBAC \
  --token-auth-file=/etc/kubernetes/ssl/basic-auth.csv \
  --bind-address=10.2.1.190 \
  --client-ca-file=/etc/kubernetes/ssl/ca.pem \
  --endpoint-reconciler-type=lease \
  --etcd-cafile=/etc/kubernetes/ssl/ca.pem \
  --etcd-certfile=/etc/kubernetes/ssl/kubernetes.pem \
  --etcd-keyfile=/etc/kubernetes/ssl/kubernetes-key.pem \
  --etcd-servers=https://10.2.1.190:2379,https://10.2.1.191:2379,https://10.2.1.192:2379 \
  --kubelet-certificate-authority=/etc/kubernetes/ssl/ca.pem \
  --kubelet-client-certificate=/etc/kubernetes/ssl/admin.pem \
  --kubelet-client-key=/etc/kubernetes/ssl/admin-key.pem \
  --service-account-issuer=kubernetes.default.svc \
  --service-account-signing-key-file=/etc/kubernetes/ssl/ca-key.pem \
  --service-account-key-file=/etc/kubernetes/ssl/ca.pem \
  --service-cluster-ip-range=10.68.0.0/16 \
  --service-node-port-range=20000-40000 \
  --tls-cert-file=/etc/kubernetes/ssl/kubernetes.pem \
  --tls-private-key-file=/etc/kubernetes/ssl/kubernetes-key.pem \
  --feature-gates=RemoveSelfLink=false \  #添加这个配置
  --requestheader-client-ca-file=/etc/kubernetes/ssl/ca.pem \
  --requestheader-allowed-names= \
  --requestheader-extra-headers-prefix=X-Remote-Extra- \
  --requestheader-group-headers=X-Remote-Group \
  --requestheader-username-headers=X-Remote-User \
  --proxy-client-cert-file=/etc/kubernetes/ssl/aggregator-proxy.pem \
  --proxy-client-key-file=/etc/kubernetes/ssl/aggregator-proxy-key.pem \
  --enable-aggregator-routing=true \
  --v=2
Restart=always
RestartSec=5
Type=notify
LimitNOFILE=65536

[Install]
WantedBy=multi-user.target

 重启服务 systemctl daemon-reload && systemctl restart kubelet

最好是整体服务器重启下文章来源地址https://www.toymoban.com/news/detail-693221.html

到了这里，关于k8s 配置hadoop集群，nfs作为存储的文章就介绍完了。如果您还想了解更多内容，请在右上角搜索TOY模板网以前的文章或继续浏览下面的相关文章，希望大家以后多多支持TOY模板网！