Create pagelist endpoint for Article API to enable Copilot consumers …

…(#52220)
github · docs-bot · Sep 16, 2024 · Sep 16, 2024 · Sep 16, 2024 · Sep 16, 2024
commit 145b7518ea5237f4ea2432eb4b1dbd5e9ade55b8
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -66,6 +66,7 @@ jobs:
           - observability
           # - open-source
           - pageinfo
+          - pagelist
           # - pages
           - products
           - redirects

diff --git a/src/frame/middleware/api.ts b/src/frame/middleware/api.ts
@@ -5,6 +5,7 @@ import events from '@/events/middleware.js'
 import anchorRedirect from '@/rest/api/anchor-redirect.js'
 import search from '@/search/middleware/search.js'
 import pageInfo from '@/pageinfo/middleware'
+import pageList from '@/pagelist/middleware'
 import webhooks from '@/webhooks/middleware/webhooks.js'
 import { ExtendedRequest } from '@/types'
 
@@ -14,6 +15,7 @@ router.use('/events', events)
 router.use('/webhooks', webhooks)
 router.use('/anchor-redirect', anchorRedirect)
 router.use('/pageinfo', pageInfo)
+router.use('/pagelist', pageList)
 
 // The purpose of this is for convenience to everyone who runs this code
 // base locally but don't have an Elasticsearch server locally.

diff --git a/src/pagelist/README.md b/src/pagelist/README.md
@@ -0,0 +1,7 @@
+# Pagelist
+
+This subject folder contains the code for the `/api/pagelist` endpoint.
+
+## What it does
+
+The `/api/pagelist` endpoint provides a flat structure of all the pages on GitHub Docs.
diff --git a/src/pagelist/middleware.ts b/src/pagelist/middleware.ts
@@ -0,0 +1,64 @@
+import express from 'express'
+import type { Response } from 'express'
+
+import type { ExtendedRequest } from '@/types'
+import { defaultCacheControl } from '@/frame/middleware/cache-control.js'
+import { getProductStringFromPath, getVersionStringFromPath } from '#src/frame/lib/path-utils.js'
+import { latest } from '#src/versions/lib/enterprise-server-releases.js'
+
+const router = express.Router()
+
+router.get('/v1/enterprise-server@latest', (req, res) => {
+  res.redirect(
+    307,
+    req.originalUrl.replace(
+      '/pagelist/v1/enterprise-server@latest',
+      `/pagelist/v1/enterprise-server@${latest}`,
+    ),
+  )
+})
+
+router.get('/v1/:product@:version', (req: ExtendedRequest, res: Response) => {
+  const { product, version } = req.params
+
+  if (!req.context || !req.context.pages) throw new Error('Request not contextualized.')
+
+  const pages = req.context.pages
+
+  // the keys of `context.pages` are permalinks
+  const keys = Object.keys(pages)
+
+  // we filter the permalinks to get only our target version
+  const filteredPermalinks = keys.filter((key) => versionMatcher(key, `${product}@${version}`))
+
+  if (!filteredPermalinks.length) {
+    res.status(400).type('text').send('Invalid version')
+    return
+  }
+
+  defaultCacheControl(res)
+
+  // new line added at the end so `wc` works as expected with `-l` and `-w`.
+  res.type('text').send(filteredPermalinks.join('\n').concat('\n'))
+})
+
+router.get('/:product@:version', (req, res) => {
+  res.redirect(307, req.originalUrl.replace('/pagelist', '/pagelist/v1'))
+})
+
+// If no version is provided we'll assume API v1 and Docs version FPT
+router.get('/', (req, res) => {
+  res.redirect(307, req.originalUrl.replace('/pagelist', '/pagelist/v1/free-pro-team@latest'))
+})
+
+function versionMatcher(key: string, targetVersion: string) {
+  const versionFromPath = getVersionStringFromPath(key)
+
+  if (!versionFromPath) {
+    throw new Error(`Couldn't get version from the permalink ${key} when generating the pagelist.`)
+  }
+  if (getProductStringFromPath(key) === 'early-access') return null
+  if (versionFromPath === targetVersion) return key
+}
+
+export default router
diff --git a/src/pagelist/tests/pagelist.ts b/src/pagelist/tests/pagelist.ts
@@ -0,0 +1,64 @@
+import { beforeAll, describe, expect, test } from 'vitest'
+
+import { get } from '#src/tests/helpers/e2etest.js'
+
+import { allVersionKeys } from '#src/versions/lib/all-versions.js'
+import nonEnterpriseDefaultVersion from '#src/versions/lib/non-enterprise-default-version.js'
+import { latest } from '#src/versions/lib/enterprise-server-releases.js'
+
+test('redirects without version suffix', async () => {
+  const res = await get(`/api/pagelist`)
+  expect(res.statusCode).toBe(307)
+  expect(res.headers.location).toBe(`/api/pagelist/v1/${nonEnterpriseDefaultVersion}`)
+})
+
+test('redirects for ghes@latest', async () => {
+  const res = await get(`/api/pagelist/v1/enterprise-server@latest`)
+  expect(res.statusCode).toBe(307)
+  expect(res.headers.location).toBe(`/api/pagelist/v1/enterprise-server@${latest}`)
+})
+
+describe.each(allVersionKeys)('pagelist api for %s', async (versionKey) => {
+  beforeAll(() => {
+    // If you didn't set the `ROOT` variable, the tests will fail rather
+    // cryptically. So as a warning for engineers running these tests,
+    // alert in case it was accidentally forgotten.
+    if (!process.env.ROOT) {
+      console.warn(
+        'WARNING: The pagelist tests require the ROOT environment variable to be set to the fixture root',
+      )
+    }
+    // Ditto for fixture-based translations to work
+    if (!process.env.TRANSLATIONS_FIXTURE_ROOT) {
+      console.warn(
+        'WARNING: The pagelist tests require the TRANSLATIONS_FIXTURE_ROOT environment variable to be set',
+      )
+    }
+  })
+
+  // queries the pagelist API for each version
+  const res = await get(`/api/pagelist/v1/${versionKey}`)
+
+  test('is reachable, returns 200 OK', async () => {
+    expect(res.statusCode).toBe(200)
+  })
+
+  // there's a large assortment of possible URLs,
+  // even "/en" is an acceptable URL, so regexes capture lots
+  test('contains valid urls', async () => {
+    let expression
+
+    // if we're testing the default version, it may be missing
+    // from the url altogether so we need a slightly different regex
+    if (versionKey === nonEnterpriseDefaultVersion)
+      expression = new RegExp(`/\\w{2}(/${versionKey})?/?.*`)
+    else expression = new RegExp(`/\\w{2}/${versionKey}/?.*`)
+
+    res.body
+      .trim()
+      .split('\n')
+      .forEach((permalink: string) => {
+        expect(permalink).toMatch(expression)
+      })
+  })
+})