audit: BreadcrumbList, robots policy, og:title, sitemap priorities

docusaurus.config.js

@@ -449,20 +449,58 @@ module.exports = {
           changefreq: "weekly",
           priority: 0.5,
           filename: "sitemap.xml",
+          // Task 35: differentiate docs sitemap priorities by content type
+          // so search engines spend crawl budget proportional to how
+          // canonical each page is. Priority buckets:
+          //   1.0  → /docs/ root (highest — primary entry point)
+          //   0.9  → /docs/quickstart/* (highest-intent user flow)
+          //   0.8  → /docs/running-keploy/* (primary product docs)
+          //   0.7  → /docs/concepts/*, /docs/keploy-explained/*
+          //   0.6  → /docs/keploy-cloud/*, /docs/ci-cd/*
+          //   0.6  → /docs/faq, /docs/troubleshooting (reference-style)
+          //   0.5  → /docs/concepts/reference/glossary/* (long-tail
+          //          glossary; noindexed legacy versions excluded via
+          //          netlify headers + robots.txt)
           createSitemapItems: async (params) => {
             const {defaultCreateSitemapItems, ...rest} = params;
             const items = await defaultCreateSitemapItems(rest);
             return items.map((item) => {
-              if (item.url.includes("/quickstart/")) {
+              const url = item.url;
+              // The /docs/ home page is the highest-priority entry point
+              // for the whole docs subtree.
+              if (url.endsWith("/docs/") || url.endsWith("/docs")) {
+                return {...item, priority: 1.0, changefreq: "weekly"};
+              }
+              if (url.includes("/quickstart/")) {
+                return {...item, priority: 0.9, changefreq: "weekly"};
+              }
+              if (url.includes("/running-keploy/")) {
                 return {...item, priority: 0.8, changefreq: "weekly"};
               }
               if (
-                item.url.includes("/concepts/") ||
-                item.url.includes("/keploy-explained/")
+                url.includes("/concepts/reference/glossary/")
+              ) {
+                // Glossary entries are numerous, long-tail, and often
+                // off-topic for core product queries. Keep them in the
+                // sitemap but mark them low priority.
+                return {...item, priority: 0.5, changefreq: "monthly"};
+              }
+              if (
+                url.includes("/concepts/") ||
+                url.includes("/keploy-explained/")
               ) {
                 return {...item, priority: 0.7, changefreq: "weekly"};
               }
-              if (item.url.includes("/keploy-cloud/")) {
+              if (
+                url.includes("/keploy-cloud/") ||
+                url.includes("/ci-cd/")
+              ) {
+                return {...item, priority: 0.6, changefreq: "monthly"};
+              }
+              if (
+                url.includes("/faq") ||
+                url.includes("/troubleshooting")
+              ) {
                 return {...item, priority: 0.6, changefreq: "monthly"};
               }
               return item;

src/theme/DocBreadcrumbs/index.js

@@ -51,9 +51,14 @@ export default function DocBreadcrumbs() {
   const {siteConfig} = useDocusaurusContext();
   const {pathname} = useLocation();
 
-  if (!breadcrumbs) {
-    return null;
-  }
+  // LIVE-20 fix. Previously this component early-returned when
+  // useSidebarBreadcrumbs() returned null/undefined, which caused
+  // glossary and reference pages not in the sidebar config to ship
+  // with zero BreadcrumbList schema (audited 2026-04-14 on
+  // /docs/concepts/reference/glossary/idempotency/).
+  // Now we treat null/undefined as "no sidebar trail, emit Home + Docs
+  // schema anyway" so AI crawlers always get a hierarchy signal.
+  const sidebarTrail = Array.isArray(breadcrumbs) ? breadcrumbs : [];
 
   const toAbsoluteUrl = (baseUrl, url) => {
     if (!url) {
@@ -89,9 +94,9 @@ export default function DocBreadcrumbs() {
     }
   }
 
-  if (breadcrumbs.length > 0) {
-    breadcrumbs.forEach((crumb, index) => {
-      const isLast = index === breadcrumbs.length - 1;
+  if (sidebarTrail.length > 0) {
+    sidebarTrail.forEach((crumb, index) => {
+      const isLast = index === sidebarTrail.length - 1;
       const href =
         crumb.type === "category" && crumb.linkUnlisted
           ? undefined
@@ -130,35 +135,37 @@ export default function DocBreadcrumbs() {
           </script>
         </Head>
       )}
-      <nav
-        className={clsx(
-          ThemeClassNames.docs.docBreadcrumbs,
-          styles.breadcrumbsContainer
-        )}
-        aria-label={translate({
-          id: "theme.docs.breadcrumbs.navAriaLabel",
-          message: "Breadcrumbs",
-          description: "The ARIA label for the breadcrumbs",
-        })}
-      >
-        <ul className="breadcrumbs">
-          {homePageRoute && <HomeBreadcrumbItem />}
-          {breadcrumbs.map((item, idx) => {
-            const isLast = idx === breadcrumbs.length - 1;
-            const href =
-              item.type === "category" && item.linkUnlisted
-                ? undefined
-                : item.href;
-            return (
-              <BreadcrumbsItem key={idx} active={isLast}>
-                <BreadcrumbsItemLink href={href} isLast={isLast}>
-                  {item.label}
-                </BreadcrumbsItemLink>
-              </BreadcrumbsItem>
-            );
+      {sidebarTrail.length > 0 && (
+        <nav
+          className={clsx(
+            ThemeClassNames.docs.docBreadcrumbs,
+            styles.breadcrumbsContainer
+          )}
+          aria-label={translate({
+            id: "theme.docs.breadcrumbs.navAriaLabel",
+            message: "Breadcrumbs",
+            description: "The ARIA label for the breadcrumbs",
           })}
-        </ul>
-      </nav>
+        >
+          <ul className="breadcrumbs">
+            {homePageRoute && <HomeBreadcrumbItem />}
+            {sidebarTrail.map((item, idx) => {
+              const isLast = idx === sidebarTrail.length - 1;
+              const href =
+                item.type === "category" && item.linkUnlisted
+                  ? undefined
+                  : item.href;
+              return (
+                <BreadcrumbsItem key={idx} active={isLast}>
+                  <BreadcrumbsItemLink href={href} isLast={isLast}>
+                    {item.label}
+                  </BreadcrumbsItemLink>
+                </BreadcrumbsItem>
+              );
+            })}
+          </ul>
+        </nav>
+      )}
     </>
   );
 }

src/theme/DocItem/index.js

@@ -147,8 +147,24 @@ export default function DocItem(props) {
   const normalizedMetaKeywords = Array.isArray(metaKeywords)
     ? metaKeywords.join(", ")
     : metaKeywords;
+  // LIVE-13: suppress Article / BlogPosting / APIReference schema on the
+  // /docs/ root and any category index pages. Article schema on a hub
+  // page is a type mismatch because a hub does not have a single author,
+  // single publication date, or single headline — it is an index of
+  // content. Hub pages emit only the normal DocBreadcrumbs JSON-LD.
+  const permalink = metadata?.permalink || "";
+  const isDocsRoot =
+    permalink === "/docs/" ||
+    permalink === "/docs" ||
+    permalink.endsWith("/docs/index") ||
+    permalink.endsWith("/docs/");
+  const isCategoryIndex =
+    frontMatter?.slug === "index" ||
+    /\/category\/|\/index\/?$/.test(permalink);
+  const suppressArticleSchema = isDocsRoot || isCategoryIndex;
+
   const articleSchema =
-    pageUrl && title
+    pageUrl && title && !suppressArticleSchema
       ? {
           "@context": "https://schema.org",
           "@type": schemaType,
@@ -187,6 +203,20 @@ export default function DocItem(props) {
         {normalizedMetaKeywords && (
           <meta name="keywords" content={normalizedMetaKeywords} />
         )}
+        {/* LIVE-12: per-page og:title and og:description override the
+            docusaurus.config.js site-level defaults, which previously
+            emitted "Keploy Documentation" as og:title on every docs
+            page regardless of content. Social card previews now reflect
+            the actual page title (e.g. "What is Idempotency in REST
+            APIs? Complete Guide"). */}
+        <meta property="og:title" content={title} />
+        {description && (
+          <meta property="og:description" content={description} />
+        )}
+        <meta name="twitter:title" content={title} />
+        {description && (
+          <meta name="twitter:description" content={description} />
+        )}
         {socialImage && <meta property="og:image" content={socialImage} />}
         {socialImage && <meta name="twitter:image" content={socialImage} />}
         {socialImage && (

static/robots.txt

@@ -1,12 +1,98 @@
-# Block specific bot
+# Keploy docs robots.txt
+# Policy: allow AI search/answer engines, block training-only crawlers,
+# block Bytespider. Search bots drive visibility in ChatGPT, Claude,
+# Perplexity, Copilot, Gemini answers. Training bots feed future model
+# weights and provide nothing back.
+# Reference: Speedscale / Katalon / Testsigma split policy (2026 competitor audit)
+
+# =============================================================================
+# ALLOW — AI search / answer engines
+# Legacy-version disallows are repeated inside this group because a bot that
+# matches a named User-agent group only reads rules from THAT group; it does
+# not fall through to `User-agent: *`. Without these lines, Perplexity/
+# Applebot/OAI-SearchBot/etc. would still crawl /docs/{1,2,3}.0.0/ despite
+# the global block further below.
+# =============================================================================
+
+User-agent: OAI-SearchBot
+User-agent: ChatGPT-User
+User-agent: Claude-SearchBot
+User-agent: Claude-User
+User-agent: PerplexityBot
+User-agent: Perplexity-User
+User-agent: Gemini-Deep-Research
+User-agent: GoogleOther
+User-agent: Applebot
+User-agent: DuckAssistBot
+User-agent: Amazonbot
+Allow: /
+Crawl-delay: 5
+Disallow: /cgi-bin/
+Disallow: /docs/1.0.0/
+Disallow: /docs/2.0.0/
+Disallow: /docs/3.0.0/
+
+# =============================================================================
+# DISALLOW — Training-only crawlers
+# =============================================================================
+
+User-agent: GPTBot
+Disallow: /
+
+User-agent: ClaudeBot
+Disallow: /
+
+User-agent: anthropic-ai
+Disallow: /
+
+User-agent: CCBot
+Disallow: /
+
+User-agent: Google-Extended
+Disallow: /
+
+User-agent: Applebot-Extended
+Disallow: /
+
+User-agent: Meta-ExternalAgent
+Disallow: /
+
+User-agent: FacebookBot
+Disallow: /
+
+User-agent: cohere-ai
+Disallow: /
+
+User-agent: Diffbot
+Disallow: /
+
+User-agent: Omgilibot
+Disallow: /
+
+User-agent: ImagesiftBot
+Disallow: /
+
+# Always-block scraper
 User-agent: Bytespider
 Disallow: /
 
-# Default rules — apply to all crawlers including AI bots
+# =============================================================================
+# DEFAULT — Googlebot, Bingbot, and all other crawlers
+# =============================================================================
+
 User-agent: *
 Allow: /
 Crawl-delay: 5
 Disallow: /cgi-bin/
 
+# Block unmaintained legacy doc versions (already set via noindex + canonical,
+# belt-and-braces for crawlers that ignore those signals).
+Disallow: /docs/1.0.0/
+Disallow: /docs/2.0.0/
+Disallow: /docs/3.0.0/
+
+# =============================================================================
 # Sitemap
+# =============================================================================
+
 Sitemap: https://keploy.io/docs/sitemap.xml

versioned_docs/version-4.0.0/keploy-explained/keploy-vs-alternatives.md

@@ -0,0 +1,72 @@
+---
+id: keploy-vs-alternatives
+title: "Keploy vs Alternatives"
+sidebar_label: Keploy vs Alternatives
+description: "Side-by-side comparison of Keploy with Postman, Katalon, WireMock, Testcontainers, and other API testing tools. Feature matrix, approach, strengths, and when to pick each."
+keywords:
+  - keploy vs postman
+  - keploy alternatives
+  - api testing tool comparison
+  - keploy vs katalon
+  - keploy vs wiremock
+  - keploy vs testcontainers
+---
+
+# Keploy vs Alternatives
+
+Keploy occupies a different point in the API testing design space than most competitors. This page is a reference comparison so you can decide which tool fits your workflow before adopting anything.
+
+The shared axis across every tool: **how do tests get created and how expensive is it to maintain them**.
+
+## Feature comparison matrix
+
+| Capability | Keploy | Postman | Katalon | WireMock | Testcontainers |
+|---|---|---|---|---|---|
+| Test generation model | Auto from real traffic (eBPF capture) | Manual scripts | Manual + low-code | Manual + record/playback | Manual + real containers |
+| SDK / code changes required | None (kernel-level eBPF) | Newman CLI integration | Groovy scripts or record | Java SDK or standalone proxy | Java / Go / Node / Python SDK |
+| Mock generation | Automatic, per-dependency | Manual per endpoint | Built-in mock server | Central mock definitions | Real container instances |
+| Non-determinism handling | Built-in (timestamps, UUIDs, tokens) | Manual regex matchers | Test data profiles | Request matcher rules | Not applicable |
+| Secret masking at capture | Automatic (Bearer, Stripe, AWS, JWT, PCI) | Manual | Manual | Manual | Not applicable |
+| CI/CD integration | GitHub Actions, GitLab, Jenkins, CircleCI | Newman in any CI | Built-in | Any JVM CI | Any CI with Docker |
+| License | Apache 2.0 (OSS) | Freemium (commercial) | Commercial | Apache 2.0 | Apache 2.0 |
+| Kernel version requirement | Linux 5.5+ (CO-RE) | N/A | N/A | N/A | N/A |
+
+## Approach differences
+
+**Keploy** captures real traffic flowing through a running service using eBPF at the Linux kernel level, then replays that traffic as deterministic tests. You point it at a staging or local instance, run the real API calls you want covered (or let real users use the app), and Keploy writes YAML test fixtures. No SDK, no proxy, no code instrumentation.
+
+**Postman** is a manual API client. Every request and every assertion has to be written by a human. It is excellent for exploratory testing and building up a contract, but it scales linearly with the number of endpoints — more endpoints means more tests to write and maintain.
+
+**Katalon** is a low-code test automation platform with a record-and-playback GUI. It reduces the amount of scripting needed compared to pure Postman, but test maintenance still tracks endpoint count.
+
+**WireMock** is a record/playback HTTP mock server. You can capture real responses and replay them, but WireMock stops at the mock boundary — it does not generate the test cases that call into the system under test. It is the "mock side" of what Keploy does on both capture and replay.
+
+**Testcontainers** spins up real instances of databases, queues, and services inside Docker for integration tests. It is the "real dependency" approach: instead of mocking Postgres, you run a real Postgres container for every test. High fidelity, high cost per test run.
+
+## When to pick each
+
+- **Pick Keploy** when you have an API-heavy service with 50+ endpoints and want regression coverage to grow automatically with usage instead of proportional to engineering time spent writing tests.
+- **Pick Postman** when you are actively developing a new API and need interactive exploration of request/response shapes during design.
+- **Pick Katalon** when you are a QA-led organization with a preference for low-code tools and want GUI record/playback as the primary workflow.
+- **Pick WireMock** when you need a lightweight standalone mock server for a small set of HTTP dependencies and you are already writing tests in Java.
+- **Pick Testcontainers** when your tests need genuine database or queue behavior that cannot be meaningfully mocked — transactions across multiple tables, time-based queries, or complex query planner behavior.
+
+Many teams combine tools: Keploy for the regression layer, Postman for exploratory development, and Testcontainers for the small number of tests where real database behavior is the point.
+
+## Migration paths
+
+If you are currently using Postman or Katalon and want to evaluate Keploy without throwing away existing work:
+
+1. Run a 15-minute capture session against your staging environment while a human exercises the endpoints you already have Postman collections for.
+2. Commit the generated `keploy/` directory to git.
+3. Run `keploy test` in CI alongside your existing Postman suite.
+4. Compare the two suites over a few sprints — which catches more regressions, which has more false positives, which takes more engineering time to maintain.
+5. If Keploy wins, retire the Postman collections gradually.
+
+The Keploy CLI can also ingest existing OpenAPI specs or Postman collections as a starting point for test generation, so you do not have to rebuild coverage from scratch.
+
+## Related reading
+
+- [How Keploy works](./how-keploy-works.md) — the eBPF capture + replay architecture in detail
+- [Keploy integration testing FAQs](./integration-testing-faq.md)
+- [Keploy API testing FAQs](./api-testing-faq.md)

versioned_sidebars/version-4.0.0-sidebars.json

@@ -166,6 +166,11 @@
           "label": "Troubleshooting Guide",
           "id": "keploy-explained/common-errors"
         },
+        {
+          "type": "doc",
+          "label": "Keploy vs Alternatives",
+          "id": "keploy-explained/keploy-vs-alternatives"
+        },
         {
           "type": "doc",
           "label": "FAQs",