name: Ingest

on:

workflow_dispatch:

schedule:

- cron: "10 8-23/3 * * *" # every 3rd hour from 8 through 23

push:

branches:

- master

paths:

- ingest/**

- plugins/**

- src/**

- static/**

- static.toml

- sidebar.js

- package.json

- yarn.lock

- tailwind.config.js

- '**/*.md'

- '**/*.mdx'

env:

# Assignees for broken links issues (all repos)

BROKEN_LINKS_ASSIGNEES: "kanelatechnical,ilyam8"

permissions:

actions: write

contents: write

issues: write

pull-requests: write

jobs:

ingest:

runs-on: ubuntu-latest

steps:

- name: Access Token

uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1

with:

token: ${{ secrets.GITHUB_TOKEN }}

fetch-depth: 0

- name: github authentication

uses: webfactory/ssh-agent@e83874834305fe9a4a2997156cb26c5de65a8555 # v0.10.0

with:

ssh-private-key: ${{ secrets.NETDATABOT_SSH_PRIVATE_KEY }}

- name: Init python env

uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0

with:

python-version: "3.13"

- name: Setup python env

run: |

python -m pip install --require-hashes -r .learn_environment/ingest-requirements.txt

- name: Resolve kickstart checksum

run: |

wget -O "$RUNNER_TEMP/kickstart.sh" https://raw.githubusercontent.com/netdata/netdata/master/packaging/installer/kickstart.sh

checksum="$(md5sum "$RUNNER_TEMP/kickstart.sh" | cut -d ' ' -f 1)"

if [[ ! "$checksum" =~ ^[0-9a-f]{32}$ ]]; then

echo "Invalid kickstart checksum: $checksum" >&2

exit 1

fi

printf 'KICKSTART_CHECKSUM=%s\n' "$checksum" >> "$GITHUB_ENV"

- name: Ingest process, integration generation and learn_link checking

id: ingest

run: |

# Run ingest and capture output

set +e

OUTPUT=$(python ingest/ingest.py --fail-links --kickstart-checksum "$KICKSTART_CHECKSUM" 2>&1)

EXIT_CODE=$?

set -e

echo "$OUTPUT"

# Save output for later steps

EOF=$(dd if=/dev/urandom bs=15 count=1 status=none | base64)

echo "output<<$EOF" >> $GITHUB_OUTPUT

echo "$OUTPUT" >> $GITHUB_OUTPUT

echo "$EOF" >> $GITHUB_OUTPUT

# Accept only final states emitted after every ingest stage completed.

RESULT=$(printf '%s' "$OUTPUT" | python ingest/classify_ingest_result.py "$EXIT_CODE")

if [ "$RESULT" = "broken_links" ]; then

echo "has_broken_links=true" >> $GITHUB_OUTPUT

elif [ "$RESULT" = "success" ]; then

echo "has_broken_links=false" >> $GITHUB_OUTPUT

else

echo "Unexpected ingest classification: $RESULT"

exit 2

fi

# Don't fail the workflow on broken links - we'll create an issue instead

exit 0

- name: Verify generated recovery state

run: |

set -o pipefail

snapshot_generated_outputs() {

local docs_manifest="$RUNNER_TEMP/generated-docs.sha256"

local fixed_manifest="$RUNNER_TEMP/generated-fixed.sha256"

if ! find docs -type f -print0 | sort -z | xargs -0 sha256sum > "$docs_manifest"; then

echo "Failed to hash the generated documentation corpus" >&2

return 1

fi

if ! sha256sum \

ingest/generated_map.yaml \

ingest/generated_sidebar_order.json \

ingest/generated_sidebar_order.json.sha256 \

netlify.toml > "$fixed_manifest"; then

echo "Failed to hash the generated recovery artifacts" >&2

return 1

fi

if ! LC_ALL=C sort "$docs_manifest" "$fixed_manifest"; then

echo "Failed to sort the generated recovery manifest" >&2

return 1

fi

}

snapshot_generated_outputs > "$RUNNER_TEMP/generated-before.sha256"

python ingest/ingest.py --regenerate-grids-only

snapshot_generated_outputs > "$RUNNER_TEMP/generated-after.sha256"

diff -u \

"$RUNNER_TEMP/generated-before.sha256" \

"$RUNNER_TEMP/generated-after.sha256"

- name: Create pull request

id: create-pr

uses: peter-evans/create-pull-request@5f6978faf089d4d20b00c7766989d076bb2fc7f1 # v8.1.1

with:

token: ${{ secrets.GITHUB_TOKEN }}

commit-message: Ingest new documentation

title: "Ingest New Documentation"

body: |

- Ingest new documentation

Auto-generated by [create-pull-request][1]

[1]: https://github.com/peter-evans/create-pull-request:

branch: ingest

labels: ingest, automation

- name: Run rendered link checks for the automated PR

if: >-

steps.create-pr.outputs.pull-request-operation == 'created' ||

steps.create-pr.outputs.pull-request-operation == 'updated'

uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0

env:

PULL_REQUEST_NUMBER: ${{ steps.create-pr.outputs.pull-request-number }}

with:

github-token: ${{ secrets.GITHUB_TOKEN }}

script: |

const pullNumber = Number(process.env.PULL_REQUEST_NUMBER);

if (!Number.isSafeInteger(pullNumber) || pullNumber <= 0) {

throw new Error(`Invalid pull-request number: ${process.env.PULL_REQUEST_NUMBER}`);

}

const { data: pull } = await github.rest.pulls.get({

owner: context.repo.owner,

repo: context.repo.repo,

pull_number: pullNumber,

});

if (pull.head.repo.full_name !== `${context.repo.owner}/${context.repo.repo}`) {

throw new Error(`Refusing to dispatch checks for external head repository ${pull.head.repo.full_name}`);

}

await github.rest.actions.createWorkflowDispatch({

owner: context.repo.owner,

repo: context.repo.repo,

workflow_id: 'rendered-link-integrity.yml',

ref: pull.head.ref,

inputs: {

pr_number: String(pull.number),

base_sha: pull.base.sha,

head_sha: pull.head.sha,

},

});

- name: Create or update broken links issue

if: steps.ingest.outputs.has_broken_links == 'true'

uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0

with:

github-token: ${{ secrets.GITHUB_TOKEN }}

script: |

const output = `${{ steps.ingest.outputs.output }}`;

const configuredAssignees = '${{ env.BROKEN_LINKS_ASSIGNEES }}'.split(',').map(s => s.trim()).filter(s => s);

const assignees = [];

for (const assignee of configuredAssignees) {

try {

await github.request('GET /repos/{owner}/{repo}/assignees/{assignee}', {

owner: context.repo.owner,

repo: context.repo.repo,

assignee,

});

assignees.push(assignee);

} catch (error) {

if (error.status === 404) {

console.warn(`Skipping unavailable issue assignee: ${assignee}`);

continue;

}

throw error;

}

}

const today = new Date().toISOString().split('T')[0];

// Build issue body with the script output

let body = `## Broken Links Detected\n\n`;

body += `**Date:** ${today}\n\n`;

body += `This issue was automatically created by the ingest workflow.\n\n`;

body += `### Ingest Output\n\n`;

body += `\`\`\`\n`;

// Extract only the broken links sections from output

const lines = output.split('\n');

let inBrokenSection = false;

let brokenOutput = [];

for (const line of lines) {

if (line.includes('### Uncorrelated URLs') || line.includes('### Broken Header/Anchor Links')) {

inBrokenSection = true;

}

if (inBrokenSection) {

brokenOutput.push(line);

}

if (line.includes('### FAILURE:')) {

break;

}

}

body += brokenOutput.join('\n');

body += `\n\`\`\`\n\n`;

body += `---\n`;

body += `> Please fix these broken links in the respective source repositories.\n`;

body += `> Once fixed, the links will be updated in Learn on the next ingest run.\n`;

// Check if there's already an open issue

const existingIssues = await github.rest.issues.listForRepo({

owner: context.repo.owner,

repo: context.repo.repo,

state: 'open',

labels: 'broken-links'

});

if (existingIssues.data.length > 0) {

// Update existing issue

const issueNumber = existingIssues.data[0].number;

await github.rest.issues.update({

owner: context.repo.owner,

repo: context.repo.repo,

issue_number: issueNumber,

body: body,

assignees: assignees

});

console.log(`Updated existing issue #${issueNumber}`);

} else {

// Create new issue

const issue = await github.rest.issues.create({

owner: context.repo.owner,

repo: context.repo.repo,

title: `[Broken Links] Documentation has broken internal links`,

body: body,

labels: ['broken-links', 'automation'],

assignees: assignees

});

console.log(`Created issue #${issue.data.number}`);

}