Azure JWT verification in Go is not working

I have a Go HTTP Server. I want to protect my routes using Azure JWT Token. I am able to generate the token but I am not able to verify it.

This is how I am doing it:

package main

import (
    "context"
    "errors"
    "fmt"

    "github.com/dgrijalva/jwt-go"
    "github.com/lestrrat-go/jwx/jwa"
    "github.com/lestrrat-go/jwx/jwk"
    njwt "github.com/lestrrat-go/jwx/jwt"
)

const token = "<access-token>"

const jwksURL = `https://login.microsoftonline.com/common/discovery/keys`

func main() {
    set, _ := jwk.Fetch(context.TODO(), jwksURL)
    // verified that set has required kid 
    verify2(token, set)
    token, err := verify(token, set)
    // token, err := jwt.Parse(token, getKey)
    if err != nil {
        panic(err)
    }
    claims := token.Claims.(jwt.MapClaims)
    for key, value := range claims {
        fmt.Printf("%st%vn", key, value)
    }
}

func verify2(token string, keyset jwk.Set) {
    btoken := []byte(token)
    parsedToken, err := njwt.Parse(
        btoken, //token is a []byte
        njwt.WithKeySet(keyset),
        njwt.WithValidate(true),
    )
    fmt.Printf("%v %v", parsedToken, err)
}

func verify(tokenString string, keySet jwk.Set) (*jwt.Token, error) {
    tkn, err := jwt.Parse(tokenString, func(token *jwt.Token) (interface{}, error) {
        if token.Method.Alg() != jwa.RS256.String() {
            return nil, fmt.Errorf("unexpected signing method: %v", token.Header["alg"])
        }
        kid, ok := token.Header["kid"].(string)
        if !ok {
            return nil, errors.New("kid header not found")
        }
        keys, ok := keySet.LookupKeyID(kid)
        if !ok {
            return nil, fmt.Errorf("key %v not found", kid)
        }
        var raw interface{}
        err := keys.Raw(&raw)
        return raw, err
    })
    return tkn, err
}

verify2(..) gives <nil> failed to match any of the keys and
verify(..) gives crypto/rsa: verification error

my JWT header:

{
  "typ": "JWT",
  "nonce": "...",
  "alg": "RS256",
  "x5t": "-KI3Q9nNR7bRofxmeZoXqbHZGew",
  "kid": "-KI3Q9nNR7bRofxmeZoXqbHZGew"
}