Provisioning an Alibaba Cloud Kubernetes (ACK) Multi-AZ Kubernetes Cluster Using Terraform

Prerequisites

  1. Terraform and providers installed according to https://www.alibabacloud.com/help/doc-detail/67384.htm
  2. Terraform 0.11.11 installed and runs on Linux
  3. Terraform AliCloud provider version 1.29
  4. Terraform Kubernetes provider version 1.5
  5. Docker image(s) is pushed to registry already
  6. A domain name is being managed in Alibaba Cloud DNS (e.g. seyantszkin.xyz)

Target Environment

Our target infrastructure includes:

  1. 1 Multi-AZ Kubernetes cluster in 2 Availability Zones (Max. 3 zones supported)
  2. 3 Kubernetes Master Nodes across 2 Availability Zones with 2 Server Load Balancer instances
  3. 3 Kubernetes Worker Nodes across 2 Availability Zones
  4. 1 NAT Gateway instance for downloading Docker image(s) from Docker Hub
  5. 1 Server Load Balancer (SLB) instance for Nginx Ingress Controller
  6. 1 Kubernetes Service serves sample web application connecting to Alibaba Cloud ApsaraDB RDS for MySQL (RDS)
  7. 1 Alibaba Cloud DNS record will be bind to the Kubernetes Service

Terraform — Infrastructure as Code

Terraform allow us to describe the target environment, and then manage the lifecycle of the environment. In this case, Terraform files are modularized into multiple files for easier management in which variables are externalized to each “config” files.

Before Provisioning

In the accesskey.tf, update the Alibaba Cloud Access Key and Secret.

variable "access_key" {
default = ""
}
variable "secret_key" {
default = ""
}
variable "region" {
default = "cn-hongkong"
}
variable "azone1" {
default = "cn-hongkong-b"
}
variable "azone2" {
default = "cn-hongkong-c"
}
variable "azone3" {
default = "cn-hongkong-c"
}
  1. ~/.kube/cluster-ca-cert.pem
  2. ~/.kube/client-key.pem
  3. ~/.kube/client-cert.pem

Start Provisioning

Execute IaC.sh or command “terraform apply –auto-approve”

resource "alicloud_vpc" "vpc" {
name = "${var.vpc_name}"
cidr_block = "${var.vpc_cidr}"
}
resource "alicloud_vswitch" "vswitch1" {
availability_zone = "${var.azone1}"
name = "${var.vswitch_name1}"
cidr_block = "${var.vswitch_cidr1}"
vpc_id = "${alicloud_vpc.vpc.id}"
}
resource "alicloud_vswitch" "vswitch2" {
availability_zone = "${var.azone1}"
name = "${var.vswitch_name2}"
cidr_block = "${var.vswitch_cidr2}"
vpc_id = "${alicloud_vpc.vpc.id}"
}
resource "alicloud_vswitch" "vswitch3" {
availability_zone = "${var.azone3}"
name = "${var.vswitch_name3}"
cidr_block = "${var.vswitch_cidr3}"
vpc_id = "${alicloud_vpc.vpc.id}"
}
resource "alicloud_nat_gateway" "nat_gateway" {
vpc_id = "${alicloud_vpc.vpc.id}"
specification = "Small"
name = "kin-k8s-tf-nat-gw"
depends_on = [
"alicloud_vswitch.vswitch1",
"alicloud_vswitch.vswitch2",
"alicloud_vswitch.vswitch3"
]
}
resource "alicloud_eip" "eip1" {
bandwidth = "20"
}
resource "alicloud_eip_association" "eip1_asso" {
allocation_id = "${alicloud_eip.eip1.id}"
instance_id = "${alicloud_nat_gateway.nat_gateway.id}"
}
resource "alicloud_snat_entry" "snat1" {
snat_table_id = "${alicloud_nat_gateway.nat_gateway.snat_table_ids}"
source_vswitch_id = "${alicloud_vswitch.vswitch1.id}"
snat_ip = "${alicloud_eip.eip1.ip_address}"
}
resource "alicloud_snat_entry" "snat2" {
snat_table_id = "${alicloud_nat_gateway.nat_gateway.snat_table_ids}"
source_vswitch_id = "${alicloud_vswitch.vswitch2.id}"
snat_ip = "${alicloud_eip.eip1.ip_address}"
}
resource "alicloud_snat_entry" "snat3" {
snat_table_id = "${alicloud_nat_gateway.nat_gateway.snat_table_ids}"
source_vswitch_id = "${alicloud_vswitch.vswitch3.id}"
snat_ip = "${alicloud_eip.eip1.ip_address}"
}
resource "alicloud_db_instance" "rdsinstance" {
engine = "MySQL"
engine_version = "5.6"
instance_type = "rds.mysql.t1.small"
instance_storage = "5"
vswitch_id = "${alicloud_vswitch.vswitch1.id}"
security_ips = ["${var.vswitch_cidr1}", "${var.vswitch_cidr2}", "${var.vswitch_cidr3}", "${alicloud_eip.eip1.ip_address}"]
#depends_on = ["alicloud_cs_kubernetes.k8s-cluster"]
}
resource "alicloud_db_database" "rdsdb" {
instance_id = "${alicloud_db_instance.rdsinstance.id}"
name = "demodb"
character_set = "utf8"
}
resource "alicloud_db_account" "mysqlroot" {
instance_id = "${alicloud_db_instance.rdsinstance.id}"
name = "${var.db_credential["username"]}"
password = "${var.db_credential["password"]}"
}
resource "alicloud_db_account_privilege" "default" {
instance_id = "${alicloud_db_instance.rdsinstance.id}"
account_name = "${alicloud_db_account.mysqlroot.name}"
privilege = "ReadWrite"
db_names = ["${alicloud_db_database.rdsdb.name}"]
}
resource "alicloud_cs_kubernetes" "k8s-cluster" {
name = "${var.k8clu_name}"
vswitch_ids = [ #Indicates Multiple Availability Zone
"${alicloud_vswitch.vswitch1.id}",
"${alicloud_vswitch.vswitch2.id}",
"${alicloud_vswitch.vswitch3.id}"
]
master_instance_types = ["${var.master_type["zone1"]}","${var.master_type["zone1"]}","${var.master_type["zone2"]}"]
master_disk_category = "cloud_efficiency" #cloud_ssd or cloud_efficiency
master_disk_size = "40"
worker_instance_types = ["${var.worker_type["zone1"]}","${var.worker_type["zone1"]}","${var.worker_type["zone2"]}"]
worker_disk_category = "cloud_efficiency" #cloud_ssd or cloud_efficiency
worker_disk_size = "40"
worker_data_disk_category = "cloud_ssd" #cloud_ssd or cloud_efficiency
worker_data_disk_size = "40"
worker_numbers = [1,1,1]
key_name = "${alicloud_key_pair.k8s-ssh-key.key_name}" #for ECS ssh key auth, either key_name or password
#password = "${var.k8ssh["password"]}" #for ECS password auth, either key_name or password
new_nat_gateway = "false"
pod_cidr = "172.20.0.0/16"
service_cidr = "172.30.0.0/16"
slb_internet_enabled = "true" #for SLB of K8S API Server
enable_ssh = "true" #SSH login kubernetes
install_cloud_monitor = "true"
cluster_network_type = "terway"
kube_config = "${var.kube_cli["cfg"]}"
client_cert = "${var.kube_cli["client_cert"]}"
client_key = "${var.kube_cli["client_key"]}"
cluster_ca_cert = "${var.kube_cli["k8s_ca"]}"
depends_on = [
"alicloud_eip_association.eip1_asso",
"alicloud_snat_entry.snat1",
"alicloud_snat_entry.snat2",
"alicloud_snat_entry.snat3"
]

}
variable "app1"{
default =
{
name = "sample"
namespace = "default"
min_replicas = 2
max_replicas = 10
cpu_threshold = 80
#image_repo = "registry-intl.ap-southeast-1.aliyuncs.com/kin-test-acr/demo"
image_repo = "seyantszkin/demo" #
image_ver = "v1.0"
svc_port = 80
container_port = 8080
svc_type = "LoadBalancer"
}
}
resource "kubernetes_deployment" "app1" {

metadata {
name = "${var.app1["name"]}"
labels {
app = "${var.app1["name"]}"
}
namespace = "${var.app1["namespace"]}"
}

spec {
replicas = "${var.app1["min_replicas"]}"

selector {
match_labels {
app = "${var.app1["name"]}"
}
}

template {
metadata {
labels {
app = "${var.app1["name"]}"
}
}
spec {
container {
env {
name = "MYSQL_HOST"
value = "${alicloud_db_instance.rdsinstance.connection_string}"
}
env {
name = "MYSQL_PORT"
value = "3306"
}
env {
name = "DB_USERNAME"
value = "${var.db_credential["username"]}"
}
env {
name = "DB_PASSWORD"
value = "${var.db_credential["password"]}"
}
image = "${var.app1["image_repo"]}:${var.app1["image_ver"]}"
name = "${var.app1["name"]}"
port {
container_port = "${var.app1["container_port"]}"
protocol = "TCP"
}
#resources {
# requests {}
#}
}
#image_pull_secrets {
# name = "${kubernetes_secret.reg_secret.metadata.0.name}"
#}

}
}

}
depends_on = [
"alicloud_cs_kubernetes.k8s-cluster",
"alicloud_db_database.rdsdb"
]
}
resource "kubernetes_horizontal_pod_autoscaler" "hpa1" {

metadata {
name = "${var.app1["name"]}"
}
spec {
max_replicas = "${var.app1["max_replicas"]}"
min_replicas = "${var.app1["min_replicas"]}"
target_cpu_utilization_percentage = "${var.app1["cpu_threshold"]}"
scale_target_ref {
kind = "Deployment"
name = "${var.app1["name"]}"
}
}
depends_on = ["kubernetes_deployment.app1"]
}
resource "kubernetes_service" "svc1" {
metadata {
name = "${var.app1["name"]}-svc"
namespace = "${var.app1["namespace"]}"
}
spec {
selector {
app = "${var.app1["name"]}"
}

port {
port = "${var.app1["svc_port"]}"
target_port = "${var.app1["container_port"]}"
protocol = "TCP"
}

session_affinity = "None"
type = "${var.app1["svc_type"]}"

}
depends_on = ["kubernetes_deployment.app1"]
}
resource "alicloud_dns_record" "svc1" {
name = "alitest.org" # A domain name (e.g. alicloud.org) registered under your Alibaba Cloud account
host_record = "${var.app1["name"]}"
type = "A"
value = "${kubernetes_service.svc1.load_balancer_ingress.0.ip}"
}

After Provisioning

The output of the Terraform should look like following screenshot:

Troubleshooting Potential Issues

Terraform is not able to repair, fix or retry in case of timeout of the provisioning processes. In addition, it is possible that metadata of Terraform may be corrupted when applying changes to the Terraform managed stack. Therefore, it is better to wrap up the provisioning process or even the Terraform command in a shell script to handle exceptions.

Summary

In this post, I have shown how Terraform can integrate resources on Alibaba Cloud with an application provisioned on ACK cluster. You can extend it to stack lifecycle management processes of your Alibaba Cloud resources.

--

--

Get the Medium app

A button that says 'Download on the App Store', and if clicked it will lead you to the iOS App store
A button that says 'Get it on, Google Play', and if clicked it will lead you to the Google Play store
Alibaba Cloud

Alibaba Cloud

Follow me to keep abreast with the latest technology news, industry insights, and developer trends. Alibaba Cloud website:https://www.alibabacloud.com