Xcode – How to locate elements in iOS UI test for flutter fastlane screnshots

Pavlo's answer is what I ended up going with. I wanted to respond to this with a more complete guide explaining what to do.

First, add these dev_dependencies to pubspec.yaml:

  flutter_test:
    sdk: flutter
  integration_test:
    sdk: flutter
  flutter_driver:
    sdk: flutter

Then make a file called test_driver/integration_driver.dart like this:

import 'dart:io';
import 'package:integration_test/integration_test_driver_extended.dart';

Future<void> main() async {
  try {
    await integrationDriver(
      onScreenshot: (String screenshotName, List<int> screenshotBytes) async {
        final File image = await File('screenshots/$screenshotName.png')
            .create(recursive: true);
        image.writeAsBytesSync(screenshotBytes);
        return true;
      },
    );
  } catch (e) {
    print('Error occured taking screenshot: $e');
  }
}

Then make a file called something like integration_test/screenshot_test.dart:

import 'dart:io';
import 'dart:ui';

import 'package:device_info/device_info.dart';
import 'package:flutter/material.dart';
import 'package:flutter_test/flutter_test.dart';
import 'package:integration_test/integration_test.dart';

import 'package:auslan_dictionary/common.dart';
import 'package:auslan_dictionary/flashcards_landing_page.dart';
import 'package:auslan_dictionary/globals.dart';
import 'package:auslan_dictionary/main.dart';
import 'package:auslan_dictionary/word_list_logic.dart';

// Note, sometimes the test will crash at the end, but the screenshots do
// actually still get taken.

Future<void> takeScreenshot(
    WidgetTester tester,
    IntegrationTestWidgetsFlutterBinding binding,
    ScreenshotNameInfo screenshotNameInfo,
    String name) async {
  if (Platform.isAndroid) {
    await binding.convertFlutterSurfaceToImage();
    await tester.pumpAndSettle();
  }
  await tester.pumpAndSettle();
  await binding.takeScreenshot(
      "${screenshotNameInfo.platformName}/en-AU/${screenshotNameInfo.deviceName}-${screenshotNameInfo.physicalScreenSize}-${screenshotNameInfo.getAndIncrementCounter()}-$name");
}

class ScreenshotNameInfo {
  String platformName;
  String deviceName;
  String physicalScreenSize;
  int counter = 1;

  ScreenshotNameInfo(
      {required this.platformName,
      required this.deviceName,
      required this.physicalScreenSize});

  int getAndIncrementCounter() {
    int out = counter;
    counter += 1;
    return out;
  }

  static Future<ScreenshotNameInfo> buildScreenshotNameInfo() async {
    Size size = window.physicalSize;
    String physicalScreenSize = "${size.width.toInt()}x${size.height.toInt()}";

    DeviceInfoPlugin deviceInfo = DeviceInfoPlugin();

    String platformName;
    String deviceName;
    if (Platform.isAndroid) {
      platformName = "android";
      AndroidDeviceInfo info = await deviceInfo.androidInfo;
      deviceName = info.product;
    } else if (Platform.isIOS) {
      platformName = "ios";
      IosDeviceInfo info = await deviceInfo.iosInfo;
      deviceName = info.name;
    } else {
      throw "Unsupported platform";
    }

    return ScreenshotNameInfo(
        platformName: platformName,
        deviceName: deviceName,
        physicalScreenSize: physicalScreenSize);
  }
}

void main() async {
  final IntegrationTestWidgetsFlutterBinding binding =
      IntegrationTestWidgetsFlutterBinding();
  binding.framePolicy = LiveTestWidgetsFlutterBindingFramePolicy.fullyLive;
  IntegrationTestWidgetsFlutterBinding.ensureInitialized();
  testWidgets("takeScreenshots", (WidgetTester tester) async {

    // Just examples of taking screenshots.
    await takeScreenshot(tester, binding, screenshotNameInfo, "search");

    final Finder searchField = find.byKey(ValueKey("searchPage.searchForm"));
    await tester.tap(searchField);
    await tester.pumpAndSettle();
    await tester.enterText(searchField, "hey");
    await takeScreenshot(tester, binding, screenshotNameInfo, "searchWithText");
  });
}

You can then invoke it like this:

flutter drive --driver=test_driver/integration_driver.dart --target=integration_test/screenshot_test.dart -d 'iPhone 13 Pro Max'

Make sure to first make the appropriate directories, like this:

mkdir -p screenshots/ios/en-AU
mkdir -p screenshots/android/en-AU

There is currently an issue with Flutter / its testing deps that as of 2.10.4 means you have to alter the testing package: https://github.com/flutter/flutter/issues/91668. In short, make the following change to packages/integration_test/ios/Classes/IntegrationTestPlugin.m:

+ (void)registerWithRegistrar:(NSObject<FlutterPluginRegistrar> *)registrar {
    [[IntegrationTestPlugin instance] setupChannels:registrar.messenger];
}

You might need to run flutter clean after this.

Now you're at the point where you can take screenshots!

As a bonus, this Python script will spin up a bunch of simulators / emulators for iOS and Android and drive the above integration test to take screenshots for all of them:

import argparse
import asyncio
import logging
import os
import re


# The list of iOS simulators to run.
# This comes from inspecting `xcrun simctl list`
IOS_SIMULATORS = [
    "iPhone 8",
    "iPhone 8 Plus",
    "iPhone 13 Pro Max",
    "iPad Pro (12.9-inch) (5th generation)",
    "iPad Pro (9.7-inch)",
]

ANDROID_EMULATORS = [
    "Nexus_7_API_32",
    "Nexus_10_API_32",
    "Pixel_5_API_32",
]


LOG = logging.getLogger(__name__)
formatter = logging.Formatter("%(asctime)s - %(levelname)s - %(message)s")
ch = logging.StreamHandler()
ch.setFormatter(formatter)
LOG.addHandler(ch)


def parse_args():
    parser = argparse.ArgumentParser()
    parser.add_argument("-d", "--debug", action="store_true")
    parser.add_argument("--clear-screenshots", action="store_true", help="Delete all existing screenshots")
    args = parser.parse_args()
    return args


class cd:
    """Context manager for changing the current working directory"""
    def __init__(self, newPath):
        self.newPath = os.path.expanduser(newPath)

    def __enter__(self):
        self.savedPath = os.getcwd()
        os.chdir(self.newPath)

    def __exit__(self, etype, value, traceback):
        os.chdir(self.savedPath)


async def run_command(command):
    proc = await asyncio.create_subprocess_exec(
        *command,
        stdout=asyncio.subprocess.PIPE,
        stderr=asyncio.subprocess.PIPE,
    )
    stdout, stderr = await proc.communicate()
    if stderr:
        LOG.debug(f"stderr of command {command}: {stderr}")
    return stdout.decode("utf-8")


async def get_uuids_of_ios_simulators(simulators):
    command_output = await run_command(["xcrun", "simctl", "list"])

    out = {}
    for s in simulators:
        for line in command_output.splitlines():
            r = "    " + re.escape(s) + r" ((.*)) (.*"
            m = re.match(r, line)
            if m is not None:
                out[s] = m[1]

    return out


async def start_ios_simulators(uuids_of_ios_simulators):
    async def start_ios_simulator(uuid):
        await run_command(["xcrun", "simctl", "boot", uuid])

    await asyncio.gather(
        *[start_ios_simulator(uuid) for uuid in uuids_of_ios_simulators.values()]
    )


async def start_android_emulators(android_emulator_names):
    async def start_android_emulator(name):
        await run_command(["flutter", "emulators", "--launch", name])

    await asyncio.gather(
        *[start_android_emulator(name) for name in android_emulator_names]
    )


async def get_all_device_ids():
    raw = await run_command(["flutter", "devices"])
    out = []
    for line in raw.splitlines():
        if "•" not in line:
            continue
        if "Daniel" in line:
            continue
        if "Chrome" in line:
            continue
        device_id = line.split("•")[1].lstrip().rstrip()
        out.append(device_id)

    return out


async def run_tests(device_ids):
    async def run_test(device_id):
        LOG.info(f"Started testing for {device_id}")
        await run_command(
            [
                "flutter",
                "drive",
                "--driver=test_driver/integration_driver.dart",
                "--target=integration_test/screenshot_test.dart",
                "-d",
                device_id,
            ]
        )
        LOG.info(f"Finished testing for {device_id}")

    for device_id in device_ids:
        await run_test(device_id)

    # await asyncio.gather(*[run_test(device_id) for device_id in device_ids])


async def main():
    args = parse_args()

    # chdir to location of python file.
    abspath = os.path.abspath(__file__)
    dname = os.path.dirname(abspath)
    os.chdir(dname)

    if args.debug:
        LOG.setLevel("DEBUG")
    else:
        LOG.setLevel("INFO")

    if args.clear_screenshots:
        await run_command(["rm", "ios/en-AU/*"])
        await run_command(["rm", "android/en-AU/*"])
        LOG.info("Cleared existing screenshots")

    uuids_of_ios_simulators = await get_uuids_of_ios_simulators(IOS_SIMULATORS)
    LOG.info(f"iOS simulatior name to UUID: {uuids_of_ios_simulators}")

    LOG.info("Launching iOS simulators")
    await start_ios_simulators(uuids_of_ios_simulators)
    LOG.info("Launched iOS simulators")

    LOG.info("Launching Android emulators")
    await start_android_emulators(ANDROID_EMULATORS)
    LOG.info("Launched Android emulators")

    await asyncio.sleep(5)

    device_ids = await get_all_device_ids()
    LOG.debug(f"Device IDs: {device_ids}")

    LOG.info("Running tests")
    await run_tests(device_ids)
    LOG.info("Ran tests")

    LOG.info("Done!")


if __name__ == "__main__":
    asyncio.run(main())

Note, if you try to take multiple screenshots on Android with this approach, you'll have a bad time. Check out https://github.com/flutter/flutter/issues/92381. The fix is still en route.