Flutter: how to make the provider rebuild only one widget

In my Flutter application I have a form with various TextFormFields, each with a microphone to transcribe what is said.
I implemented it with SpeechToTextProvider.

When I push a microphone, it writes to the right input, however all microphones activate and the setState updates all inputs and not just the one selected by the user.

How can I ensure that only one microphone is selected and rebuilt at a time?

Thanks to anyone who can help me!

This is the initialization of the the speechProvider in the root:

final SpeechToText speech = SpeechToText();
  late SpeechToTextProvider speechProvider;

  @override
  void initState() {
    super.initState();
    speechProvider = SpeechToTextProvider(speech);
    initSpeechState();
  }

  Future<void> initSpeechState() async {
    await speechProvider.initialize();
  }

Then I add the the ChangeNotifierProvider in the root widget:

ChangeNotifierProvider<SpeechToTextProvider>.value(value: speechProvider),

This is the form:

class DayForm extends StatefulWidget {
  const DayForm({required this.days, super.key});

  final Map<int, Day> days;

  @override
  State<DayForm> createState() => _DayFormState();
}

class _DayFormState extends State<DayForm> {
  Map<int, Day> get days => widget.days;

  final _formKey = GlobalKey<FormState>(debugLabel: "DayForm");

  late final Map<int, dynamic> _daysFields = {};

  void _setLastTextChange(inputKey, lastText) {
    setState(() {
      _daysFields[inputKey]["lastTextChange"] = lastText;
    });
  }

  @override
  void initState() {
    super.initState();

    for (int dayNumber = 1; dayNumber <= 7; dayNumber++) {
      _daysFields[dayNumber] = {
        "controller": TextEditingController(),
        "lastTextChange": "",
      };
    }

    days.forEach((key, value) {
      _daysFields[key]["controller"].text = value.description ?? "";
      _setLastTextChange(key, _daysFields[key]["controller"].text);
    });
  }

  @override
  Widget build(BuildContext context) {
    var formFields = <Widget>[];

    _daysFields.forEach((key, value) {
      formFields.add(Padding(
        padding: const EdgeInsets.all(16.0),
        child: Row(
          key: UniqueKey(), // here or in Padding is the same
          children: [
            Expanded(
              child: TextFormField(
                controller: value["controller"],
                onChanged: (changed) {
                  _setLastTextChange(key, changed);
                },
              ),
            ),
            SpeechProvider(
              key: ValueKey(key),
              controller: value["controller"],
              lastInputChange: value["lastTextChange"],
              setLastInputChange: _setLastTextChange,
            ),
          ],
        ),
      ));
    });

    return Consumer<ApplicationState>(
      builder: (context, appState, _) {
        return Form(
          key: _formKey,
          child: SingleChildScrollView(
            child: Column(
              children: formFields,
            ),
          ),
        );
      },
    );
  }
}

And here is the code to call SpeechToTextProvider:

import 'package:flutter/material.dart';
import 'package:provider/provider.dart';
import 'package:speech_to_text/speech_recognition_result.dart';
import 'package:speech_to_text/speech_to_text_provider.dart';

class SpeechProvider extends StatefulWidget {
  const SpeechProvider({
    super.key,
    required this.controller,
    required this.lastInputChange,
    required this.setLastInputChange,
  });

  final TextEditingController controller;
  final String lastInputChange;
  final void Function(dynamic, String) setLastInputChange;

  @override
  SpeechProviderState createState() => SpeechProviderState();
}

class SpeechProviderState extends State<SpeechProvider> {
  String lastWords = '';
  late SpeechToTextProvider speechProvider;

  void startListening() {
    if (speechProvider.isAvailable && speechProvider.isNotListening) {
      lastWords = '';

      speechProvider.listen();

      speechProvider.stream.listen((event) {
        final result = event.recognitionResult;
        if (result != null) {
          resultListener(result);
        }
      });

      setState(() {});
    }
  }

  void stopListening() {
    if (!mounted) return;
    if (speechProvider.isListening) {
      speechProvider.stop();
      setState(() {
        widget.setLastInputChange((widget.key as ValueKey<dynamic>).value, widget.controller.text);
      });
    }
  }

  void resultListener(SpeechRecognitionResult result) {
    if (!mounted) return;

    setState(() {
      if (result.finalResult) {
        widget.setLastInputChange((widget.key as ValueKey<dynamic>).value, widget.controller.text);
      } else {
        lastWords = result.recognizedWords;
        widget.controller.text = '${widget.lastInputChange} $lastWords';
      }
    });
  }

  @override
  Widget build(BuildContext context) {
    speechProvider = Provider.of<SpeechToTextProvider>(context);

    return TapRegion(
      onTapOutside: (tap) => speechProvider.isListening ? stopListening() : null,
      child: SizedBox(
        child: FloatingActionButton(
          onPressed: speechProvider.isListening ? stopListening : startListening,
          child: Icon(speechProvider.isListening ? Icons.mic : Icons.mic_off),
        ),
      ),
    );
  }
}