Paginate data with query cursors

SarbjitSingh
June 26, 2023
81 views
1 vote
2 Answers

`export const firestoreCollectionDocumentsJoin = (firestore: AngularFirestore, joins: Join[], clazz: any) => {
return (source: Observable) => defer(() => {

let documents: any[];
const cache = new Map();

return source.pipe(
  distinctUntilChanged((x, y) => {

    const x1 = x.filter((dca: DocumentChangeAction<unknown>) => {
      return !dca.payload.doc.metadata.hasPendingWrites;
    }).map((dca: DocumentChangeAction<unknown>) => {
      return set(dca.payload.doc.data() as any, 'id', dca.payload.doc.id);
    });

    const y1 = y.filter((dca: DocumentChangeAction<unknown>) => {
      return !dca.payload.doc.metadata.hasPendingWrites;
    }).map((dca: DocumentChangeAction<unknown>) => {
      return set(dca.payload.doc.data() as any, 'id', dca.payload.doc.id);
    });

    const c = x.map((dca: DocumentChangeAction<unknown>) => {
      return dca.payload.doc.metadata.fromCache;
    }).filter((d: any) => !!d);


    // return x.length === y.length;
    // console.log(`CCCCCCC`, c);

    return false;

    // return JSON.stringify(x1) === JSON.stringify(y1);

  }),
  // pipe(filter((c) => c.payload.doc.metadata.fromCache === false)),
  switchMap((_dca: DocumentChangeAction<DocumentData>[]) => {

    cache.clear();
    const reads$ = [];
    let i = 0;

    documents = _dca.map(dca => {
      let doc = set(dca.payload.doc.data(), 'id', dca.payload.doc.id);
      if (!environment.production) {
      // console.log('doc ', doc);
      }
      doc = set(doc, 'metadata', dca.payload.doc.metadata);
      return doc;
    });

    for (const doc of documents) {

      for (const join of joins) {

        let docIds = get(doc, join.id);

        if (isString(docIds)) {
          docIds = [docIds];
        }

        if (isArray(docIds)) {

          for (const docId of docIds) {

            if (isString(docId)) {

              if (cache.has(join.uniqueId(docId))) {
                continue;
              }

              cache.set(join.uniqueId(docId), i);
              i++;

              reads$.push(
                firestore
                  .collection(join.collection)
                  .doc(docId)
                  .snapshotChanges()
                  .pipe(
                    map(dca => {
                    
                      return set(dca.payload.data() as any, 'id', dca.payload.id);
                    }),
                    distinctUntilChanged((p, c) => {
                      return JSON.stringify(p) === JSON.stringify(c);
                    }),
                  )
              );
            }
          }
        }
      }
    }

    return reads$.length ? combineLatest(reads$) : of([]);
  }),
  map((values) => {
    return documents.map(document => {

      for (const join of joins) {
        const data = get(document, join.id);

        let docIds = [];
        if (isString(data)) {
          docIds = [data];
        } else if (isArray(data)) {
          docIds = data;
        }

        if (isArray(docIds)) {
          for (const docId of docIds) {
            if (isString(docId)) {
              const uniqueId = join.uniqueId(docId);
              const index = cache.get(uniqueId);
              const docData = merge(values[index], set(values[index] as object, 'id', docId));

              let currentDocData = get(document, join.data);
              if (currentDocData) {
                currentDocData = merge(currentDocData, new join.clazz(docData));
                if (isArray(currentDocData)) {
                  document[join.data].push(new join.clazz(docData));
                } else {
                  document = merge(document, set(document, join.data, currentDocData));
                }
              } else {
                if (isArray(data)) {
                  document[join.data] = [new join.clazz(docData)];
                } else if (isString(data)) {
                  document = merge(document, set(document, join.data, new join.clazz(docData)));
                }
              }
            }
          }
        }
      }
      if (!environment.production) {
     console.log('document ', new clazz(document));
      }

      return new clazz(document);
    });
  }),
  pipe(distinctUntilChanged((p, c) => {
    return JSON.stringify(p) === JSON.stringify(c);
  })),
  tap(docs => {
    if (!environment.production) {
      console.log(
        `Queried ${docs ? (docs as any).length : 0}, Joined ${cache.size} docs.`
      );
    }
  })
);

});
};`

I have tried to add filter collections with a field confirmed_booking_time and use same with startAfter for next page.
Can any one suggest how to use query cursors in paginate for firebase query.

Answers

Chosen as BEST ANSWER

I rewrite the code and this solve my issue

 public allCompletedBooking2(filter: BookingsFilter): Observable<{ bookings: Booking[], count: number }> {
    const joins = [
      new Join('client_id', 'client', PROFILES, Client),
      new Join('assigned_team_ids', 'teams', PROFILES, Team),
    ];

    // Fetch the total count of bookings in a separate query without pagination
    let count = 0;
    const countQuery = this.firestore.collection('mode')
      .doc(environment.node)
      .collection('bookings', (ref: CollectionReference) => {
        let queryFn = ref.orderBy('confirmed_booking_time');  // Order by the timestamp of each booking
        switch (filter.getTab()) {
          case BookingsFilter.TABS.CANCELED_BOOKINGS:
            queryFn = queryFn.where('is_canceled', '==', true);
            break;
          case BookingsFilter.TABS.UNCONFIRMED_BOOKINGS:
            queryFn = queryFn.where('is_confirmed', '==', false);
            break;
          case BookingsFilter.TABS.UPCOMING_BOOKINGS:
            queryFn = queryFn.where('confirmed_booking_time', '>', new Date());
            break;
          case BookingsFilter.TABS.BOOKINGS_IN_PROGRESS:
            queryFn = queryFn.where('confirmed_booking_time', '<=', new Date());
            queryFn = queryFn.where(`status`, '==', 0);
            break;
          case BookingsFilter.TABS.COMPLETED_BOOKINGS:
            // queryFn = queryFn.where('confirmed_booking_time', '<=', new Date());
            queryFn = queryFn.where(`status`, '==', 1);
            break;
          case BookingsFilter.TABS.AMENDMENTS:
            queryFn = queryFn.where('has_amendments', '==', true);
            break;
          case BookingsFilter.TABS.TODAY:
            // Get the current date as a Firestore timestamp
            const todayTimestamp = firebase.firestore.Timestamp.fromDate(moment().startOf('day').toDate());
            const todayTimestampEndOfDay = firebase.firestore.Timestamp.fromDate(moment().endOf('day').toDate());
            // Create a query to filter bookings created today and not canceled
            queryFn = queryFn
              .where('confirmed_booking_time', '>=', todayTimestamp)
              .where('confirmed_booking_time', '<', todayTimestampEndOfDay)
            // .where('is_canceled', '==', false);
            break;

          case BookingsFilter.TABS.THIS_MONTH:
            // Get the current date as a Firestore timestamp
            const startOfMonth = firebase.firestore.Timestamp.fromDate(moment().startOf('month').toDate());
            const endOfMonth = firebase.firestore.Timestamp.fromDate(moment().endOf('month').toDate());
            queryFn = queryFn
              .where('confirmed_booking_time', '>=', startOfMonth)
              .where('confirmed_booking_time', '<=', endOfMonth)
            // .where('is_canceled', '==', false);
            break;
          case BookingsFilter.TABS.OUTSTANDING_INVOICES:
           // queryFn = queryFn.where('is_canceled', '==', false);
            queryFn = queryFn.where('is_paid', '==', false);
            break;
            case BookingsFilter.TABS.PAID_INVOICES:
           // queryFn = queryFn.where('is_canceled', '==', false);
            queryFn = queryFn.where('is_paid', '==', true);
            break;
          case BookingsFilter.TABS.WAITING_FOR_CLIENT:
            queryFn = queryFn.where('is_canceled', '==', false);
            queryFn = queryFn.where('is_confirmed', '==', true);
            for (const service_id of [DSLR_PHOTOGRAPHY, VT_PHOTOGRAPHY]) {
              queryFn = queryFn.where(`jobs.${service_id}.status`, '==', 1);
              queryFn = queryFn.where(`jobs.${service_id}.has_selected_photos`, '==', false);

            }

            break;
          case BookingsFilter.TABS.WAITING_FOR_APPROVAL:

            for (const serviceId of [DSLR_PHOTOGRAPHY, VT_PHOTOGRAPHY, CAD_POST_PRODUCTION]) {

              queryFn = queryFn.where(`jobs.${serviceId}.approved_by_admin`, '==', true);
              queryFn = queryFn.where(`jobs.${serviceId}.status`, '==', 1);
              //queryFn = this.applyBaseFiltersToQuery(queryFn, filter);


            }
            break;
        }
        return queryFn;
      })
      .snapshotChanges()
      .pipe(
        firestoreCollectionDocumentsJoin(this.firestore, joins, Booking),
        tap((bookings) => {
          count = bookings.length;
        }),
      );

    // Store the last document from each page
    let lastDoc = null;
    // Return a function that fetches the next page of results when called
    let query = this.firestore.collection('mode')
      .doc(environment.node)
      .collection('bookings', (ref: CollectionReference) => {
        let queryFn = ref.orderBy('confirmed_booking_time');  // Order by the timestamp of each booking
        switch (filter.getTab()) {
          case BookingsFilter.TABS.CANCELED_BOOKINGS:
            queryFn = queryFn.where('is_canceled', '==', true);
            break;
          case BookingsFilter.TABS.UNCONFIRMED_BOOKINGS:
            queryFn = queryFn.where('is_confirmed', '==', false);
            break;
          case BookingsFilter.TABS.UPCOMING_BOOKINGS:
            queryFn = queryFn.where('confirmed_booking_time', '>', new Date());
            break;
          case BookingsFilter.TABS.BOOKINGS_IN_PROGRESS:
            queryFn = queryFn.where('confirmed_booking_time', '<=', new Date());
            queryFn = queryFn.where(`status`, '==', 0);
            break;
          case BookingsFilter.TABS.COMPLETED_BOOKINGS:
            // queryFn = queryFn.where('confirmed_booking_time', '<=', new Date());
            queryFn = queryFn.where(`status`, '==', 1);
            break;
          case BookingsFilter.TABS.AMENDMENTS:
            queryFn = queryFn.where('has_amendments', '==', true);
            break;
          case BookingsFilter.TABS.TODAY:
            // Get the current date as a Firestore timestamp
            const todayTimestamp = firebase.firestore.Timestamp.fromDate(moment().startOf('day').toDate());
            const todayTimestampEndOfDay = firebase.firestore.Timestamp.fromDate(moment().endOf('day').toDate());
            // Create a query to filter bookings created today and not canceled
            queryFn = queryFn
              .where('confirmed_booking_time', '>=', todayTimestamp)
              .where('confirmed_booking_time', '<', todayTimestampEndOfDay)
            // .where('is_canceled', '==', false);
            break;

          case BookingsFilter.TABS.THIS_MONTH:
            // Get the current date as a Firestore timestamp
            const startOfMonth = firebase.firestore.Timestamp.fromDate(moment().startOf('month').toDate());
            const endOfMonth = firebase.firestore.Timestamp.fromDate(moment().endOf('month').toDate());
            queryFn = queryFn
              .where('confirmed_booking_time', '>=', startOfMonth)
              .where('confirmed_booking_time', '<=', endOfMonth)
            // .where('is_canceled', '==', false);
            break;
          case BookingsFilter.TABS.OUTSTANDING_INVOICES:
           // queryFn = queryFn.where('is_canceled', '==', false);
            queryFn = queryFn.where('is_paid', '==', false);
            break;
            case BookingsFilter.TABS.PAID_INVOICES:
             // queryFn = queryFn.where('is_canceled', '==', false);
              queryFn = queryFn.where('is_paid', '==', true);
              break;
          case BookingsFilter.TABS.WAITING_FOR_CLIENT:
            queryFn = queryFn.where('is_canceled', '==', false);
            queryFn = queryFn.where('is_confirmed', '==', true);
            for (const service_id of [DSLR_PHOTOGRAPHY, VT_PHOTOGRAPHY]) {
              queryFn = queryFn.where(`jobs.${service_id}.status`, '==', 1);
              queryFn = queryFn.where(`jobs.${service_id}.has_selected_photos`, '==', false);

            }
            break;

          case BookingsFilter.TABS.WAITING_FOR_APPROVAL:

            for (const serviceId of [DSLR_PHOTOGRAPHY, VT_PHOTOGRAPHY, CAD_POST_PRODUCTION]) {

              queryFn = queryFn.where(`jobs.${serviceId}.approved_by_admin`, '==', true);
              queryFn = queryFn.where(`jobs.${serviceId}.status`, '==', 1);
              //queryFn = this.applyBaseFiltersToQuery(queryFn, filter);
            }
            break;
        }
        // If this is not the first page, start after the last document from the last page
        if (filter.next_page) {
          console.log('applyPaginationToQuery startAfter ', filter.start_after);
          queryFn = queryFn.startAfter(filter.start_after);
        }
        if (filter.pervious_page) {
          console.log('applyPaginationToQuery startAt ', filter.start_at);
          queryFn = queryFn.startAt(filter.start_at);
        }
        // Fetch 10 documents per page
        if (filter.getLimit()) {
          queryFn = queryFn.limit(filter.getLimit());
        }
        return queryFn;
      });

    const bookingsObservable = query.snapshotChanges()
      .pipe(
        firestoreCollectionDocuments(this.firestore, Booking),
        tap((bookings) => {
          // Store the last document from this page
          if (bookings.length > 0) {
            lastDoc = bookings[bookings.length - 1].id;
            console.log('lastDoc ', lastDoc);
          }
          // Perform actions with the filtered bookings
        })
      );

    return combineLatest([bookingsObservable, countQuery]).pipe(
      map(([bookings, _]) => ({ bookings, count })),
    );
    
  }

(Edit)

First of all, take a look to the firebase documentation: Firebase Query Cursor

After you understand how it works, take a look at this implementation:

async fetchAllBy({
    filter,
    pagination = { rows: 10, first: 0, page: 1 },
    cursor
  }: {
    filter?: FirebaseQuery<T>[];
    pagination?: Pagination;
    cursor?: QueryDocumentSnapshot<DocumentData>;
  }): Promise<FirebaseResponse<T>> {
    let queryRef = query(
      collection(this.firestore, this.collection),
      where('isDeleted', '==', false),
      orderBy('createdAt' as keyof BaseEntity, 'desc'),
      limit(pagination.rows || 10)
    );

    if (cursor) {
      queryRef = query(queryRef, startAfter(cursor!));
    }

    if (filter && filter.length > 0) {
      filter.forEach((f) => {
        queryRef = query(queryRef, where(f.key as string, f.operator, f.value));
      });
    }

    const snap = await getDocs(queryRef);
    cursor = snap.docs[snap.docs.length - 1];
    const data = snap.docs.map((doc) => doc.data() as T);
    return { data, cursor };
}

As you can see, one of the parameters is a cursor?: QueryDocumentSnapshot<DocumentData>; which we also return. The purpose is to filter the data based on the returned cursor, if one is returned. If one is not returned, we just filter the data with no cursor. Finally, return the cursor with the latest reference point.

Please signup or login to give your own answer.

Click here to cancel reply.

Paginate data with query cursors | Firebase

Answers