Next.js dynamic page not being crawled by SEO

Marshall
June 16, 2023
247 views
0 votes
2 Answers

I have a blog website in Next.Js that has a dynamic page for this route: article/[slug]. I’m using this screaming frog to crawl and index the pages and although initially before setting a custom domain, the articles were getting indexed, now the only page that appears is the homepage:

I have a robots.txt file, a sitemap generator component, I’ve run npm run build and deploy on Vercel.

Here’s [slug] component:

import AppIcon from "components/AppIcon/AppIcon";
import AppImage from "components/AppImage/AppImage";
import Loading from "components/Loading/Loading";
import PageContainer from "components/PageContainer/PageContainer";
import { ARTICLE_QUERY, graphcms, SLUGLIST } from "pages/api/graphQL/main";
import { useEffect, useState } from "react";
import parser from "react-html-parser";
import styles from "styles/articlepage.module.scss";
import { convertDate } from "utils";
import { getArticleWithGoogleAds } from "utils/googleAds";
import { jost } from "assets/fonts/nextFonts";
import NotFound from "pages/404";
import Head from "components/AppHead/AppHead";
import { useRouter } from "next/router";

function Article({ post: articleAPI }) {
  const router = useRouter();

  const [article, setArticle] = useState();

  useEffect(() => {
    if (!articleAPI) {
      return;
    }

    const articleConverted = getArticleWithGoogleAds(articleAPI);

    setArticle(articleConverted);
  }, [articleAPI]);

  useEffect(() => {
    try {
      (window.adsbygoogle = window.adsbygoogle || []).push({});
    } catch (e) {
      return console.log(e);
    }
  }, []);

  if (!article) {
    return (
      <PageContainer>
        <Loading />
      </PageContainer>
    );
  }

  if (!articleAPI && router.isFallback) {
    return <NotFound />;
  }

  return (
    <>
      <Head
        title={article.title}
        description={article.description}
        image={article.coverPhoto.url}
      />

      <PageContainer>
        <div className={styles.container} style={jost.style}>
          <AppIcon
            icon="arrow-left"
            size={30}
            color="grey"
            className={styles.arrowBack}
            onClick={() => window.history.back()}
          />
          <div>
            {article ? (
              <>
                <div>
                  <h1 className={styles.title}>{article.title}</h1>
                  <div className={styles.subtitle}>
                    <p>{convertDate(article.createdAt)}</p> <span>|</span>{" "}
                    <p>{article.category}</p>
                  </div>
                </div>
                <AppImage
                  className={styles.headerImage}
                  src={article.coverPhoto.url}
                />
                <div className={styles.content}>
                  {parser(article.content.html)}
                </div>
              </>
            ) : (
              <Loading />
            )}
          </div>
        </div>
      </PageContainer>
    </>
  );
}

export default Article;

export async function getStaticPaths() {
  const { posts } = await graphcms.request(SLUGLIST);

  const paths = posts.map((post) => {
    return { params: { slug: post.slug } };
  });

  return {
    paths,
    fallback: false
  };
}

export async function getStaticProps({ params }) {
  const slug = params.slug;

  const data = await graphcms.request(ARTICLE_QUERY, { slug });

  const post = data.post;

  return {
    props: { post: post },
    revalidate: 10
  };
}

Here’s AppHead file:

import PropTypes from "prop-types";
import Head from "next/head";
import React from "react";

function AppHead({
  title,
  description,
  image = "https://media.graphassets.com/m3c024qER0udkPRLgxOI",
  slug
}) {
  return (
    <Head>
      <meta name="description" content={description} key="desc" />
      <meta property="og:title" content={title} />
      <meta property="og:description" content={description} />
      <meta property="og:image" content={image} />
      <meta name="robots" content="/robots.txt" />
      {slug && (
        <>
          <meta
            property="og:url"
            content={`https://curiositygem.com/article/${slug}`}
          />
          <link
            rel="canonical"
            href={`https://curiositygem.com/article/${slug}`}
          />
        </>
      )}
      <meta property="og:site_name" content="curiositygem.com" />
      <meta name="twitter:description" content={description} />
      <meta name="twitter:title" content={title} />
      <meta name="viewport" content="width=device-width, initial-scale=1" />
      <title>{title}</title>
    </Head>
  );
}

AppHead.propTypes = {
  description: PropTypes.string.isRequired,
  title: PropTypes.string.isRequired,
  image: PropTypes.string,
  slug: PropTypes.string
};

export default AppHead;

Here’s _app file:

import "bootstrap/dist/css/bootstrap.min.css";
import ArticlesProvider from "context/articles-context";
import "styles/global.scss";
import Layout from "../components/Layout/Layout";

export default function App({ Component, pageProps }) {
  return (
    <ArticlesProvider>
      <Layout>
        <Component {...pageProps} />
      </Layout>
    </ArticlesProvider>
  );
}

Here’s the sitemap file:

import { graphcms, SLUGLIST } from "./api/graphQL/main";

function generateSiteMap(posts) {
  return `<?xml version="1.0" encoding="UTF-8"?>
   <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
     ${posts
       .map(
         ({ slug }) => `
       <url>
           <loc>${`https://curiositygem.com/article/${slug}`}</loc>
           <lastmod>${new Date().toISOString()}</lastmod>
           <changefreq>monthly</changefreq>
           <priority>1.0</priority>
       </url>`
       )
       .join("")}
   </urlset>
 `;
}

function SiteMap() {
  return null;
}

export async function getServerSideProps({ res }) {
  // We make an API call to gather the URLs for our site
  const data = await graphcms.request(SLUGLIST);
  const posts = data["posts"];

  // We generate the XML sitemap with the posts data
  const sitemap = generateSiteMap(posts);

  res.setHeader("Cache-Control", "s-maxage=30, stale-while-revalidate");
  res.setHeader("Content-Type", "text/xml");
  // we send the XML to the browser
  res.write(sitemap);
  res.end();

  return {
    props: {}
  };
}

export default SiteMap;

Here’s the build response, with the articles:

I’ve no idea what to do next :/

Answers

Chosen as BEST ANSWER
- Marshall
- March 9, 2023 at 12:39 am
- 0 votes
0
I've found the solution!

This was happening because of the way I was getting the articles on the homepage. When the web crawler hits the homepage, he needs to execute Javascript from my custom hook useArticles to get the posts from context, and since web crawlers don't run javascript, he got stuck on loading, not reaching the links of the posts and therefore not rendering each post route.

I hope this helps you in the future :)