skip to Main Content

I am trying to use Terraform to create an AKS cluster, then deploy a basic loadbalacer and finally get Nginx-ingress-controller to use the Loadbalancer.

Here is my Terraform code so far,

# Set default name prefix
variable "name_prefix" {
  default = "Dev"
}

# Set default location
variable "location" {
  default = "australiacentral"
}

# Create Resource Group
resource "azurerm_resource_group" "aks_rg" {
  name     = "${var.name_prefix}-rg"
  location = var.location
}

# Managed Cluster resource group
#export MC_RESOURCE_GROUP=MC_${RESOURCE_GROUP}_${AKS_CLUSTER}_${LOCATION}

resource "azurerm_kubernetes_cluster" "dk_dev_cluster" {
  name                = var.name_prefix
  location            = azurerm_resource_group.aks_rg.location
  resource_group_name = azurerm_resource_group.aks_rg.name
  dns_prefix = "Dev"
  kubernetes_version = "1.25.5"
  # Create a basic load-balancer to lower costs
  network_profile {
    network_plugin = "azure"
    load_balancer_sku = "basic"
  }

  identity {
    type = "SystemAssigned"
  }

  default_node_pool {
    name    = lower("${var.name_prefix}nodep")
    vm_size = "Standard_B2s"
    node_count = 1
    enable_auto_scaling = false
  }

  tags = {
    Environment = "Dev"
  }
}

resource "azurerm_public_ip" "lb_pub_ip" {
  name                = "${var.name_prefix}-lb-ip"
  location            = var.location
  allocation_method   = "Static"
  sku = "Basic"
  resource_group_name = azurerm_kubernetes_cluster.dk_dev_cluster.node_resource_group
}

resource "azurerm_lb" "aks_lb" {
  name                = "${var.name_prefix}-lb"
  sku                 = "Basic"
  location            = var.location
  resource_group_name = azurerm_kubernetes_cluster.dk_dev_cluster.node_resource_group
  frontend_ip_configuration {
    name                 = "PublicIPAddress"
    public_ip_address_id = azurerm_public_ip.lb_pub_ip.id
  }
}

output "client_certificate" {
  value     = azurerm_kubernetes_cluster.dk_dev_cluster.kube_config.0.client_certificate
  sensitive = true
}

output "kube_config" {
  value = azurerm_kubernetes_cluster.dk_dev_cluster.kube_config_raw

  sensitive = true
}

# Create a kubeconfig file that can be added to/replace ~/.kube/config
resource "local_file" "kubeconfig" {
  depends_on   = [azurerm_kubernetes_cluster.dk_dev_cluster]
  filename     = "kubeconfig"
  content      = azurerm_kubernetes_cluster.dk_dev_cluster.kube_config_raw
}

provider "kubernetes" {
  config_path = local_file.kubeconfig.filename
  config_context = "Dev"
}

resource "kubernetes_namespace" "nginx_namespace" {
  metadata {
    name = "nginx-ingress"
  }
  depends_on = [local_file.kubeconfig]
}

provider "helm" {
}

resource "helm_release" "nginx_ingress" {
  name       = "nginx-ingress-controller"

  repository = "https://kubernetes.github.io/ingress-nginx"
  chart      = "ingress-nginx"
  version    = "4.5.2"
  namespace = kubernetes_namespace.nginx_namespace.id

  set {
    name  = "controller.service.loadBalancerIP"
    value = azurerm_public_ip.lb_pub_ip.ip_address
  }
}

When the look at services in the nginx-ingress namespace the LoadBalacer external IP is stuck as <pending>

NAME                                                          TYPE           CLUSTER-IP    EXTERNAL-IP   PORT(S)                      AGE
nginx-ingress-controller-ingress-nginx-controller             LoadBalancer   10.0.60.72    <pending>     80:32183/TCP,443:31864/TCP   27s

kubectl get events:

41s         Warning   SyncLoadBalancerFailed   service/nginx-ingress-controller-ingress-nginx-controller                 Error syncing load balancer: failed to ensure load balancer: instance not found

From my understanding this is because nginx cannot reach the load balancer as the AKS cluster is in the non Managed resource group, while the LB is in the Manged resource group. I cannot figure out a way to deploy them into the same resource group.

Please let me know how I can allow Nginx to connect to the Loadbalancer.

2

Answers


  1. Chosen as BEST ANSWER

    I was able to solve this without adding my own Service Principal.

    To provision a AKS cluster with a Standard LB then get Nginx to use it, I used the following Terraform:

    # Set default name prefix
    variable "name_prefix" {
      default = "Dev"
    }
    
    # Set default location
    variable "location" {
      default = "australiacentral"
    }
    
    # Create Resource Group
    resource "azurerm_resource_group" "aks_rg" {
      name     = "${var.name_prefix}-rg"
      location = var.location
    }
    
    resource "azurerm_kubernetes_cluster" "dk_dev_cluster" {
      name                = var.name_prefix
      location            = azurerm_resource_group.aks_rg.location
      resource_group_name = azurerm_resource_group.aks_rg.name
      dns_prefix          = "Dev"
      kubernetes_version  = "1.25.5"
      # Create a basic load-balancer to lower costs
      network_profile {
        network_plugin    = "kubenet"
        load_balancer_sku = "standard"
      }
    
      identity {
        type = "SystemAssigned"
      }
    
      default_node_pool {
        name                = lower("${var.name_prefix}nodep")
        vm_size             = "Standard_B2s"
        node_count          = 1
        enable_auto_scaling = false
        os_sku              = "CBLMariner"
      }
    
      tags = {
        Environment = "Dev"
      }
      sku_tier = "Free"
    }
    
    resource "azurerm_public_ip" "lb_pub_ip" {
      name                = "${var.name_prefix}-lb-ip"
      location            = var.location
      allocation_method   = "Static"
      sku                 = "Standard"
      resource_group_name = azurerm_kubernetes_cluster.dk_dev_cluster.node_resource_group
    }
    
    output "client_certificate" {
      value     = azurerm_kubernetes_cluster.dk_dev_cluster.kube_config.0.client_certificate
      sensitive = true
    }
    
    output "kube_config" {
      value     = azurerm_kubernetes_cluster.dk_dev_cluster.kube_config_raw
      sensitive = true
    }
    
    output "aks_lb_public_ip" {
      value     = azurerm_public_ip.lb_pub_ip.ip_address
      sensitive = false
    }
    
    output "node_resource_group" {
      value     = azurerm_kubernetes_cluster.dk_dev_cluster.node_resource_group
      sensitive = false
    }
    
    # Create a kubeconfig file that can be added to/replace ~/.kube/config
    resource "local_file" "kubeconfig" {
      filename = "kubeconfig"
      content  = azurerm_kubernetes_cluster.dk_dev_cluster.kube_config_raw
    }
    

    Nginx ingress can then be installed using:

    
    ```shell
    NAMESPACE=ingress-nginx
    helm install ingress-nginx ingress-nginx/ingress-nginx 
    --create-namespace 
    --namespace $NAMESPACE 
    --set controller.service.annotations."service.beta.kubernetes.io/azure-load-balancer-health-probe-request-path"=/healthz 
    --set controller.service.loadBalancerIP=<PUBLIC_IP>
    

  2. I have followed below steps to create application with Nginx Ingress Controller in AKS Cluster

    When the look at services in the nginx-ingress namespace the LoadBalacer external IP is stuck as <pending>

    If Load Balacer external IP is stuck as pending state cause of below reasons. follow Stack Link for more details.

    1. Insufficient permission in service principal.

    2. Make sure you have sufficient quota to provision Public IPs in the particular region.

    You can assign Service Principal using below code.

    data "azuread_service_principal" "samplesp" {
      display_name  = azurerm_kubernetes_cluster.kubernetes_cluster[0].name
    }
    
    resource "azurerm_role_assignment" "akssp_network_contributor_subnet" {
      scope                = data.azurerm_subnet.aks-subnet.id
      role_definition_name = "Network Contributor"
      principal_id         = data.azuread_service_principal.aks-sp.object_id
    
      depends_on = [data.azuread_service_principal.aks-sp]
    }
    

    Once complete the AKS Cluster provisioning, I have deployed an application in AKS Cluster using Nginx Ingress Service.

    To check the application created with Ingress Controller use below command.

    enter image description here

    When I try to access the application using Public IP, its running successfully as below.

    enter image description here

    Refer: Kubernetes Ingress with NGINX Ingress Controller created by @Jack Roper

    Login or Signup to reply.
Please signup or login to give your own answer.
Back To Top
Search