Kubernetes【K8S】(二)：搭建Kubernetes环境

系统初始化

设置系统时区

# 设置系统时区为 亚洲/上海
[root@k8s-master01 ~]# timedatectl set-timezone Asia/Shanghai
# 设置当前得UTC时间写入硬件时钟
[root@k8s-master01 ~]# timedatectl set-local-rtc 0
# 重启依赖于系统时间的服务
[root@k8s-master01 ~]# systemctl restart rsyslog
[root@k8s-master01 ~]# systemctl restart crond

关闭邮件服务

systemctl stop postfix && systemctl disable postfix

将默认日志设置systemd journald

mkdir /var/log/journal
mkdir /etc/systemd/journald.conf.d

cat > /etc/systemd/journald.conf.d/99-prophet.conf <<-EOF
[Journal]
# 持久化保存到磁盘
Storage=persistent

# 压缩历史日志
Compress=yes

SyncIntervalSec=5m
RateLimitInterval=30s
RateLimitBurst=1000

# 最大占用空间 10G
SystemMaxUse=10G

# 单日志文件最大 200M
SystemMaxFileSize=200M

# 日志保存时间2周
MaxRetentionSec=2week

# 不将日志转发到syslog
ForwardToSyslog=no
EOF

systemctl restart systemd-journald

升级系统内核

[root@k8s-master01 ~]# yum install -y https://www.elrepo.org/elrepo-release-7.el7.elrepo.noarch.rpm
[root@k8s-master01 ~]# yum --enablerepo=elrepo-kernel install -y kernel-lt
# 查看当前系统所有内核
[root@k8s-master01 ~]# cat /boot/grub2/grub.cfg |grep "menuentry"
# 设置开机从新内核启动(版本为上步查询中的内核版本)
[root@k8s-master01 ~]# grub2-set-default 'CentOS Linux (4.4.219-1.el7.elrepo.x86_64) 7 (Core)'
# 验证是否设置成功
[root@k8s-master01 ~]# grub2-editenv list
saved_entry=CentOS Linux (4.4.219-1.el7.elrepo.x86_64) 7 (Core)
# 查看是否是新内核(重启生效)
[root@k8s-master01 ~]# uname -r
4.4.219-1.el7.elrepo.x86_64

更新yum源

# 可选
[root@k8s-master01 ~]# yum update
# 重新指定启动内核
[root@k8s-master01 ~]# cat /boot/grub2/grub.cfg |grep "menuentry"
[root@k8s-master01 ~]# grub2-set-default 'CentOS Linux (4.4.219-1.el7.elrepo.x86_64) 7 (Core)'

注意，更新yum源会将系统内核还原会初始版本，需重新设置系统内核。

使用kubeadm安装kubernetes_v1.17.5

检查网络

[root@k8s-node1 ~]# ip route show
default via 10.0.2.1 dev eth0 proto dhcp metric 100 
10.0.2.0/24 dev eth0 proto kernel scope link src 10.0.2.7 metric 100 
192.168.0.0/24 dev eth1 proto kernel scope link src 192.168.0.100 metric 101 
[root@k8s-node1 ~]# ip addr
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
    inet6 ::1/128 scope host 
       valid_lft forever preferred_lft forever
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
    link/ether 08:00:27:cd:af:47 brd ff:ff:ff:ff:ff:ff
    inet 10.0.2.7/24 brd 10.0.2.255 scope global noprefixroute dynamic eth0
       valid_lft 964sec preferred_lft 964sec
    inet6 fe80::a00:27ff:fecd:af47/64 scope link 
       valid_lft forever preferred_lft forever
3: eth1: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
    link/ether 08:00:27:21:13:37 brd ff:ff:ff:ff:ff:ff
    inet 192.168.0.100/24 brd 192.168.0.255 scope global noprefixroute eth1
       valid_lft forever preferred_lft forever
    inet6 fe80::a00:27ff:fe21:1337/64 scope link 
       valid_lft forever preferred_lft forever

节点hosts相互解析

vi /etc/hosts

10.0.2.15    k8s-node1
10.0.2.8     k8s-node2
10.0.2.9     k8s-node3

安装docker及kubelet

# 在 master 节点和 worker 节点都要执行
# 最后一个参数 1.18.4 用于指定 kubenetes 版本，支持所有 1.18.x 版本的安装
# 腾讯云 docker hub 镜像
# export REGISTRY_MIRROR="https://mirror.ccs.tencentyun.com"
# DaoCloud 镜像
# export REGISTRY_MIRROR="http://f1361db2.m.daocloud.io"
# 华为云镜像
# export REGISTRY_MIRROR="https://05f073ad3c0010ea0f4bc00b7105ec20.mirror.swr.myhuaweicloud.com"
# 阿里云 docker hub 镜像
export REGISTRY_MIRROR=https://registry.cn-hangzhou.aliyuncs.com
# 创建install_kubelet.sh脚本, 赋权755
./install_kubelet.sh -s 1.18.3
# 创建set_mirror.sh脚本赋权755

install_kubelet.sh

#!/bin/bash

# 在 master 节点和 worker 节点都要执行

# 安装 docker
# 参考文档如下
# https://docs.docker.com/install/linux/docker-ce/centos/ 
# https://docs.docker.com/install/linux/linux-postinstall/

# 卸载旧版本
yum remove -y docker 
docker-client 
docker-client-latest 
docker-ce-cli 
docker-common 
docker-latest 
docker-latest-logrotate 
docker-logrotate 
docker-selinux 
docker-engine-selinux 
docker-engine

# 设置 yum repository
yum install -y yum-utils 
device-mapper-persistent-data 
lvm2
yum-config-manager --add-repo http://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo

# 安装并启动 docker
yum install -y docker-ce-19.03.8 docker-ce-cli-19.03.8 containerd.io
systemctl enable docker
systemctl start docker

# 安装 nfs-utils
# 必须先安装 nfs-utils 才能挂载 nfs 网络存储
yum install -y nfs-utils
yum install -y wget

# 关闭 防火墙
systemctl stop firewalld
systemctl disable firewalld

# 关闭 SeLinux
setenforce 0
sed -i "s/SELINUX=enforcing/SELINUX=disabled/g" /etc/selinux/config

# 关闭 swap
swapoff -a
yes | cp /etc/fstab /etc/fstab_bak
cat /etc/fstab_bak |grep -v swap > /etc/fstab

# 修改 /etc/sysctl.conf
# 如果有配置，则修改
sed -i "s#^net.ipv4.ip_forward.*#net.ipv4.ip_forward=1#g"  /etc/sysctl.conf
sed -i "s#^net.bridge.bridge-nf-call-ip6tables.*#net.bridge.bridge-nf-call-ip6tables=1#g"  /etc/sysctl.conf
sed -i "s#^net.bridge.bridge-nf-call-iptables.*#net.bridge.bridge-nf-call-iptables=1#g"  /etc/sysctl.conf
sed -i "s#^net.ipv6.conf.all.disable_ipv6.*#net.ipv6.conf.all.disable_ipv6=1#g"  /etc/sysctl.conf
sed -i "s#^net.ipv6.conf.default.disable_ipv6.*#net.ipv6.conf.default.disable_ipv6=1#g"  /etc/sysctl.conf
sed -i "s#^net.ipv6.conf.lo.disable_ipv6.*#net.ipv6.conf.lo.disable_ipv6=1#g"  /etc/sysctl.conf
sed -i "s#^net.ipv6.conf.all.forwarding.*#net.ipv6.conf.all.forwarding=1#g"  /etc/sysctl.conf
# 可能没有，追加
echo "net.ipv4.ip_forward = 1" >> /etc/sysctl.conf
echo "net.bridge.bridge-nf-call-ip6tables = 1" >> /etc/sysctl.conf
echo "net.bridge.bridge-nf-call-iptables = 1" >> /etc/sysctl.conf
echo "net.ipv6.conf.all.disable_ipv6 = 1" >> /etc/sysctl.conf
echo "net.ipv6.conf.default.disable_ipv6 = 1" >> /etc/sysctl.conf
echo "net.ipv6.conf.lo.disable_ipv6 = 1" >> /etc/sysctl.conf
echo "net.ipv6.conf.all.forwarding = 1"  >> /etc/sysctl.conf
# 执行命令以应用
sysctl -p

# 配置K8S的yum源
cat <<EOF > /etc/yum.repos.d/kubernetes.repo
[kubernetes]
name=Kubernetes
baseurl=http://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64
enabled=1
gpgcheck=0
repo_gpgcheck=0
gpgkey=http://mirrors.aliyun.com/kubernetes/yum/doc/yum-key.gpg
       http://mirrors.aliyun.com/kubernetes/yum/doc/rpm-package-key.gpg
EOF

# 卸载旧版本
yum remove -y kubelet kubeadm kubectl

# 安装kubelet、kubeadm、kubectl
# 将 ${1} 替换为 kubernetes 版本号，例如 1.17.2
yum install -y kubelet-${1} kubeadm-${1} kubectl-${1}

# 修改docker Cgroup Driver为systemd
# # 将/usr/lib/systemd/system/docker.service文件中的这一行 ExecStart=/usr/bin/dockerd -H fd:// --containerd=/run/containerd/containerd.sock
# # 修改为 ExecStart=/usr/bin/dockerd -H fd:// --containerd=/run/containerd/containerd.sock --exec-opt native.cgroupdriver=systemd
# 如果不修改，在添加 worker 节点时可能会碰到如下错误
# [WARNING IsDockerSystemdCheck]: detected "cgroupfs" as the Docker cgroup driver. The recommended driver is "systemd". 
# Please follow the guide at https://kubernetes.io/docs/setup/cri/
sed -i "s#^ExecStart=/usr/bin/dockerd.*#ExecStart=/usr/bin/dockerd -H fd:// --containerd=/run/containerd/containerd.sock --exec-opt native.cgroupdriver=systemd#g" /usr/lib/systemd/system/docker.service

# 设置 docker 镜像，提高 docker 镜像下载速度和稳定性
# 如果您访问 https://hub.docker.io 速度非常稳定，亦可以跳过这个步骤
./set_mirror.sh -s ${REGISTRY_MIRROR}

# 重启 docker，并启动 kubelet
systemctl daemon-reload
systemctl restart docker
systemctl enable kubelet && systemctl start kubelet

docker version

set_mirror.sh

#!/usr/bin/env bash
set -e

if [ -z "$1" ]
then
    echo 'Error: Registry-mirror url required.'
    exit 1
fi

MIRROR_URL=$1
lsb_dist=''
command_exists() {
    command -v "$@" > /dev/null 2>&1
}
if command_exists lsb_release; then
    lsb_dist="$(lsb_release -si)"
    lsb_version="$(lsb_release -rs)"
fi
if [ -z "$lsb_dist" ] && [ -r /etc/lsb-release ]; then
    lsb_dist="$(. /etc/lsb-release && echo "$DISTRIB_ID")"
    lsb_version="$(. /etc/lsb-release && echo "$DISTRIB_RELEASE")"
fi
if [ -z "$lsb_dist" ] && [ -r /etc/debian_version ]; then
    lsb_dist='debian'
fi
if [ -z "$lsb_dist" ] && [ -r /etc/fedora-release ]; then
    lsb_dist='fedora'
fi
if [ -z "$lsb_dist" ] && [ -r /etc/os-release ]; then
    lsb_dist="$(. /etc/os-release && echo "$ID")"
fi
if [ -z "$lsb_dist" ] && [ -r /etc/centos-release ]; then
    lsb_dist="$(cat /etc/*-release | head -n1 | cut -d " " -f1)"
fi
if [ -z "$lsb_dist" ] && [ -r /etc/redhat-release ]; then
    lsb_dist="$(cat /etc/*-release | head -n1 | cut -d " " -f1)"
fi
lsb_dist="$(echo $lsb_dist | cut -d " " -f1)"
docker_version="$(docker -v | awk '{print $3}')"
docker_major_version="$(echo $docker_version| cut -d "." -f1)"
docker_minor_version="$(echo $docker_version| cut -d "." -f2)"
lsb_dist="$(echo "$lsb_dist" | tr '[:upper:]' '[:lower:]')"

set_daemon_json_file(){
    DOCKER_DAEMON_JSON_FILE="/etc/docker/daemon.json"
    if sudo test -f ${DOCKER_DAEMON_JSON_FILE}
    then
        sudo cp  ${DOCKER_DAEMON_JSON_FILE} "${DOCKER_DAEMON_JSON_FILE}.bak"
        if sudo grep -q registry-mirrors "${DOCKER_DAEMON_JSON_FILE}.bak";then
            sudo cat "${DOCKER_DAEMON_JSON_FILE}.bak" | sed -n "1h;1"'!'"H;${g;s|"registry-mirrors":s*[[^]]*]|"registry-mirrors": ["${MIRROR_URL}"]|g;p;}" | sudo tee ${DOCKER_DAEMON_JSON_FILE}
        else
            sudo cat "${DOCKER_DAEMON_JSON_FILE}.bak" | sed -n "s|{|{"registry-mirrors": ["${MIRROR_URL}"],|g;p;" | sudo tee ${DOCKER_DAEMON_JSON_FILE}
        fi
    else
        sudo mkdir -p "/etc/docker"
        sudo echo "{"registry-mirrors": ["${MIRROR_URL}"]}" | sudo tee ${DOCKER_DAEMON_JSON_FILE}
    fi
}


can_set_json(){
	if [ "$docker_major_version" -eq 1 ] && [ "$docker_minor_version" -lt 12 ] 
	then
		echo "docker version < 1.12"
		return 0
	else
		echo "docker version >= 1.12"
		return 1
	fi
}

restart_docker () {
    echo "systemctl daemon-reload"
    systemctl daemon-reload
    echo "systemctl restart docker"
    systemctl restart docker
    echo
    echo -e "33[31;1m--------请检查下面输出结果中的 Registry Mirrors 是否已经修改过来-------- 33[0m"
    echo "docker info"
    docker info
}

set_mirror(){
    if [ "$docker_major_version" -eq 1 ] && [ "$docker_minor_version" -lt 9 ]
        then
            echo "please upgrade your docker to v1.9 or later"
            exit 1
    fi

    case "$lsb_dist" in
        centos)
        if grep "CentOS release 6" /etc/redhat-release > /dev/null
        then
            DOCKER_SERVICE_FILE="/etc/sysconfig/docker"
            sudo cp ${DOCKER_SERVICE_FILE} "${DOCKER_SERVICE_FILE}.bak"
            sudo sed -i "s|other_args="|other_args="--registry-mirror='${MIRROR_URL}'|g" ${DOCKER_SERVICE_FILE}
            sudo sed -i "s|OPTIONS='|OPTIONS='--registry-mirror='${MIRROR_URL}'|g" ${DOCKER_SERVICE_FILE}
            echo "Success."
            restart_docker
            exit 0
        fi
        if grep "CentOS Linux release 7" /etc/redhat-release > /dev/null
        then
            if can_set_json; then
                DOCKER_SERVICE_FILE="/lib/systemd/system/docker.service"
                sudo cp ${DOCKER_SERVICE_FILE} "${DOCKER_SERVICE_FILE}.bak"
                sudo sed -i "s|(ExecStart=/usr/bin/docker[^ ]* daemon)|1 --registry-mirror="${MIRROR_URL}"|g" ${DOCKER_SERVICE_FILE}
                sudo systemctl daemon-reload
            else
                set_daemon_json_file
            fi
            echo "Success."
            restart_docker
            exit 0
        else
            echo "Error: Set mirror failed, please set registry-mirror manually please."
            exit 1
        fi
    ;;
        fedora)
        if grep "Fedora release" /etc/fedora-release > /dev/null
        then
            if can_set_json; then
            DOCKER_SERVICE_FILE="/lib/systemd/system/docker.service"
            sudo cp ${DOCKER_SERVICE_FILE} "${DOCKER_SERVICE_FILE}.bak"
            sudo sed -i "s|(ExecStart=/usr/bin/docker[^ ]* daemon)|1 --registry-mirror="${MIRROR_URL}"|g" ${DOCKER_SERVICE_FILE}
            sudo systemctl daemon-reload
            else
                set_daemon_json_file
            fi
            echo "Success."
            restart_docker
            exit 0
        else
            echo "Error: Set mirror failed, please set registry-mirror manually please."
            exit 1
        fi
    ;;
        ubuntu)
        v1=`echo ${lsb_version} | cut -d "." -f1`
        if [ "$v1" -ge 16 ]; then
            if can_set_json; then
                DOCKER_SERVICE_FILE="/lib/systemd/system/docker.service"
                sudo cp ${DOCKER_SERVICE_FILE} "${DOCKER_SERVICE_FILE}.bak"
                sudo sed -i "s|(ExecStart=/usr/bin/docker[^ ]* daemon -H fd://$)|1 --registry-mirror="${MIRROR_URL}"|g" ${DOCKER_SERVICE_FILE}
                sudo systemctl daemon-reload
            else
                set_daemon_json_file
            fi
            echo "Success."
            echo "You need to restart docker to take effect: sudo systemctl restart docker.service"
            exit 0
        else
            if can_set_json; then
                DOCKER_SERVICE_FILE="/etc/default/docker"
                sudo cp ${DOCKER_SERVICE_FILE} "${DOCKER_SERVICE_FILE}.bak"
                if grep "registry-mirror" ${DOCKER_SERVICE_FILE} > /dev/null
                then
                    sudo sed -i -u -E "s#--registry-mirror='?((http|https)://)?[a-zA-Z0-9.]+'?#--registry-mirror='${MIRROR_URL}'#g" ${DOCKER_SERVICE_FILE}
                else
                    echo 'DOCKER_OPTS="$DOCKER_OPTS --registry-mirror='${MIRROR_URL}'"' >> ${DOCKER_SERVICE_FILE}
                    echo ${MIRROR_URL}
                fi
            else
                set_daemon_json_file
            fi
        fi
        echo "Success."
        restart_docker
        exit 0
    ;;
        debian)
        if can_set_json; then
            DOCKER_SERVICE_FILE="/etc/default/docker"
            sudo cp ${DOCKER_SERVICE_FILE} "${DOCKER_SERVICE_FILE}.bak"
            if grep "registry-mirror" ${DOCKER_SERVICE_FILE} > /dev/null
            then
                sudo sed -i -u -E "s#--registry-mirror='?((http|https)://)?[a-zA-Z0-9.]+'?#--registry-mirror='${MIRROR_URL}'#g" ${DOCKER_SERVICE_FILE}
            else
                echo 'DOCKER_OPTS="$DOCKER_OPTS --registry-mirror='${MIRROR_URL}'"' >> ${DOCKER_SERVICE_FILE}
                echo ${MIRROR_URL}
            fi
        else
            set_daemon_json_file
        fi
        echo "Success."
        restart_docker
        exit 0
    ;;
        arch)
        if grep "Arch Linux" /etc/os-release > /dev/null
        then
            if can_set_json; then
                DOCKER_SERVICE_FILE="/lib/systemd/system/docker.service"
                sudo cp ${DOCKER_SERVICE_FILE} "${DOCKER_SERVICE_FILE}.bak"
                sudo sed -i "s|(ExecStart=/usr/bin/docker[^ ]* daemon -H fd://)|1 --registry-mirror="${MIRROR_URL}"|g" ${DOCKER_SERVICE_FILE}
                sudo systemctl daemon-reload
            else
                set_daemon_json_file
            fi
            echo "Success."
            restart_docker
            exit 0
        else
            echo "Error: Set mirror failed, please set registry-mirror manually please."
            exit 1
        fi
    ;;
        suse)
        if grep "openSUSE Leap" /etc/os-release > /dev/null
        then
            if can_set_json; then
            DOCKER_SERVICE_FILE="/usr/lib/systemd/system/docker.service"
            sudo cp ${DOCKER_SERVICE_FILE} "${DOCKER_SERVICE_FILE}.bak"
            sudo sed -i "s|(^ExecStart=/usr/bin/docker daemon -H fd://)|1 --registry-mirror="${MIRROR_URL}"|g" ${DOCKER_SERVICE_FILE}
            sudo systemctl daemon-reload
            else
                set_daemon_json_file
            fi
            echo "Success."
            restart_docker
            
            exit 0
        else
            echo "Error: Set mirror failed, please set registry-mirror manually please."
            exit 1
        fi
    esac
    echo "Error: Unsupported OS, please set registry-mirror manually."
    exit 1
}

set_mirror

将此shell拷贝到各节点，赋权755。每个节点都需要执行。

初始化Master节点

# 只在 master 节点执行
# 替换 x.x.x.x 为 master 节点实际 IP（请使用内网 IP）
# export 命令只在当前 shell 会话中有效，开启新的 shell 窗口后，如果要继续安装过程，请重新执行此处的 export 命令
export MASTER_IP=10.0.2.15
# 替换 apiserver.demo 为 您想要的 dnsName
export APISERVER_NAME=k8s-node1
# Kubernetes 容器组所在的网段，该网段安装完成后，由 kubernetes 创建，事先并不存在于您的物理网络中
export POD_SUBNET=10.100.0.1/16

# init_master.sh脚本赋权755
./init_master.sh -s 1.18.3

init_master.sh

#!/bin/bash

# 只在 master 节点执行

# 脚本出错时终止执行
set -e

if [ ${#POD_SUBNET} -eq 0 ] || [ ${#APISERVER_NAME} -eq 0 ]; then
  echo -e "33[31;1m请确保您已经设置了环境变量 POD_SUBNET 和 APISERVER_NAME 33[0m"
  echo 当前POD_SUBNET=$POD_SUBNET
  echo 当前APISERVER_NAME=$APISERVER_NAME
  exit 1
fi


# 查看完整配置选项 https://godoc.org/k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta2
rm -f ./kubeadm-config.yaml
cat <<EOF > ./kubeadm-config.yaml
apiVersion: kubeadm.k8s.io/v1beta2
kind: ClusterConfiguration
kubernetesVersion: v${1}
imageRepository: registry.cn-hangzhou.aliyuncs.com/google_containers
controlPlaneEndpoint: "${APISERVER_NAME}:6443"
networking:
  serviceSubnet: "10.96.0.0/16"
  podSubnet: "${POD_SUBNET}"
  dnsDomain: "cluster.local"
EOF

# kubeadm init
# 根据您服务器网速的情况，您需要等候 3 - 10 分钟
kubeadm init --config=kubeadm-config.yaml --upload-certs

# 配置 kubectl
rm -rf /root/.kube/
mkdir /root/.kube/
cp -i /etc/kubernetes/admin.conf /root/.kube/config

# 安装网络插件, 选择一项
# --------------------------------------calico-3.13.1----------------------------------
# 安装 calico 网络插件
# 参考文档 https://docs.projectcalico.org/v3.13/getting-started/kubernetes/self-managed-onprem/onpremises
echo "安装calico-3.13.1"
rm -f calico-3.13.1.yaml
wget https://kuboard.cn/install-script/calico/calico-3.13.1.yaml
kubectl apply -f calico-3.13.1.yaml

# --------------------------------------flannel----------------------------------------
# 安装 flannel 网络插件
#echo "安装flannel"
#wget https://raw.githubusercontent.com/coreos/flannel/master/Documentation/kube-flannel.yml
#kubectl apply -f kube-flannel.yml

检查初始化结果

# 只在 master 节点执行

# 执行如下命令，等待 3-10 分钟，直到所有的容器组处于 Running 状态
watch kubectl get pod -n kube-system -o wide

# 查看 master 节点初始化结果
kubectl get nodes -o wide

初始化worker节点

获取join命令参数

# 只在 master 节点执行
kubeadm token create --print-join-command

有效时间

该 token 的有效时间为 2 个小时，2小时内，您可以使用此 token 初始化任意数量的 worker 节点。

在非master节点执行kubeadm token create 输出的命令

# kubeadm token create 命令的输出
kubeadm join apiserver.demo:6443 --token mpfjma.4vjjg8flqihor4vt     --discovery-token-ca-cert-hash sha256:6f7a8e40a810323672de5eee6f4d19aa2dbdb38411845a1bf5dd63485c43d303

检查初始化结果

# 只在 master 节点执行
kubectl get nodes -o wide

参考文档: https://www.kuboard.cn/install/install-k8s.html