Modern CMake Cross Compiling to AArch64 with Sysroots – Ubuntu

Consider the following example project_(CMakeLists.txt):

cmake_minimum_required(VERSION 3.1)

project(CCL LANGUAGES C CXX)

set(CMAKE_CXX_STANDARD 17)
set(CMAKE_CXX_STANDARD_REQUIRED ON)
set(CMAKE_C_STANDARD 99)
set(CMAKE_C_STANDARD_REQUIRED ON)

find_package(PkgConfig REQUIRED)
find_package(ZLIB)
find_package(PNG)
find_library(MATH_LIBRARY m)
pkg_search_module(OpenEXR OpenEXR)
add_executable(main main.cpp)
if (MATH_LIBRARY)
  target_link_libraries(main PUBLIC ${MATH_LIBRARIES})
endif()

Main.cpp:

#include <iostream>
#include <cmath>

int main(void)
{
    std::cout << "Hello, sin()" << std::sin(30) << std::endl;
    return 0;
}

I want to compile this project with the following CMake toolchain
(aarch64-toolchain.cmake):

# Cross-compilation system information.
set(CMAKE_SYSTEM_NAME Linux)
set(CMAKE_SYSTEM_PROCESSOR aarch64)

# The sysroot contains all the libraries we might need to link against and
# possibly headers we need for compilation.
set(CMAKE_SYSROOT /var/lib/schroot/chroots/ubuntu-focal-arm64)
set(CMAKE_FIND_ROOT_PATH ${CMAKE_SYSROOT})
set(CMAKE_LIBRARY_ARCHITECTURE aarch64-linux-gnu)

# Install path when SYSROOT is read-only.
# set(CMAKE_STAGING_PREFIX aarch64-staging)

# Set the compilers for C, C++ and Fortran.
set(GCC_TRIPLE "aarch64-linux-gnu")
set(CMAKE_C_COMPILER ${GCC_TRIPLE}-gcc-10 CACHE FILEPATH "C compiler")
set(CMAKE_CXX_COMPILER ${GCC_TRIPLE}-g++-10 CACHE FILEPATH "C++ compiler")
set(CMAKE_Fortran_COMPILER ${GCC_TRIPLE}-gfortran CACHE FILEPATH "Fortran compiler")

# Automatically use the cross-wrapper for pkg-config when available.
set(PKG_CONFIG_EXECUTABLE aarch64-linux-gnu-pkg-config CACHE FILEPATH "pkg-config executable")

# Set the architecture-specific compiler flags.
set(ARCH_FLAGS "-mcpu=cortex-a53+crc+simd")
set(CMAKE_C_FLAGS_INIT ${ARCH_FLAGS})
set(CMAKE_CXX_FLAGS_INIT ${ARCH_FLAGS})
set(CMAKE_Fortran_FLAGS_INIT ${ARCH_FLAGS})

# Don't look for programs in the sysroot (these are ARM programs, they won't run
# on the build machine).
set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER)

# Only look for libraries, headers and packages in the sysroot, don't look on
# the build machine.
set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY)
set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY)
set(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE ONLY)

set(CPACK_DEBIAN_PACKAGE_ARCHITECTURE arm64)

Where the sysroot (/var/lib/schroot/chroots/ubuntu-focal-arm64) was setup using:

name=ubuntu-focal
mk-sbuild --arch=arm64 --skip-proposed --skip-updates --skip-security --name=${name} focal
su - $USER
mk-sbuild --arch=arm64 --skip-proposed --skip-updates --skip-security --name=${name} focal

Configuration-wise, this works fine, however when I try to build this project
find_library ‘correctly’ finds the wrong libm.so library:

$ cmake -DCMAKE_TOOLCHAIN_FILE=../aarch-toolchain.cmake ..
$ make
Scanning dependencies of target main
[ 50%] Building CXX object CMakeFiles/main.dir/main.cpp.o
make[2]: *** No rule to make target '/var/lib/schroot/chroots/ubuntu-focal-arm64/usr/lib/aarch64-linux-gnu/libm.so', needed by 'main'.  Stop.
make[1]: *** [CMakeFiles/Makefile2:76: CMakeFiles/main.dir/all] Error 2
make: *** [Makefile:84: all] Error 2

Looking into the sysroot itself, the library is correctly found, but obviously,
it is a symlink to a file local to the sysroot, not the host:

$ ls -hal /var/lib/schroot/chroots/ubuntu-focal-arm64/usr/lib/aarch64-linux-gnu/
lrwxrwxrwx  1 root root   32 Apr 14  2020 libm.so -> /lib/aarch64-linux-gnu/libm.so.6

It seems to me that CMake is picking up the wrong library (find_library should
really be using the compiler libraries first), so I guess that my toolchain-file
is written incorrectly somehow. How should it be changed to correctly find the
math library, without changing the sysroot or project itself?

(Also, please note that this is an example project to illustrate the problem. I still need to be able to search the sysroot for packages.)

Edits

As requested in the comments, the end state should be a proper aarch64 binary
("main"). Essentially, the build commands below should succeed both with and
without the toolchain, and should yield a functional binary (although, the
aarch64 one naturally only work inside the sysroot).

$ mkdir -p host-build && cd host-build && cmake .. && make 
$ mkdir -p device-build && cd device-build && cmake -DCMAKE_TOOLCHAIN_FILE=../aarch64-toolchain.cmake .. && make