From 19e1bb01584bcc69bfdc61ab98d980ef1bdef1c4 Mon Sep 17 00:00:00 2001 From: akmhmgc Date: Mon, 16 Nov 2026 13:53:18 +0900 Subject: [PATCH] feat(schema): add a Fields tab listing Avro fields and their docs Show every field of an Avro schema version, nested ones included, with its path, type and "doc", so field meanings can be read without going through the raw schema JSON. Co-Authored-By: Claude Opus 5.5 --- .../containers/Schema/SchemaDetail/Schema.jsx | 15 +- .../SchemaFields/SchemaFields.jsx | 142 +++++++++++ .../SchemaFields/SchemaFields.test.jsx | 94 +++++++ .../SchemaDetail/SchemaFields/index.jsx | 3 + client/src/utils/avroFields.js | 179 ++++++++++++++ client/src/utils/avroFields.test.js | 232 ++++++++++++++++++ 6 files changed, 663 insertions(+), 2 deletions(-) create mode 100644 client/src/containers/Schema/SchemaDetail/SchemaFields/SchemaFields.jsx create mode 100644 client/src/containers/Schema/SchemaDetail/SchemaFields/SchemaFields.test.jsx create mode 100644 client/src/containers/Schema/SchemaDetail/SchemaFields/index.jsx create mode 100644 client/src/utils/avroFields.js create mode 100644 client/src/utils/avroFields.test.js diff --git a/client/src/containers/Schema/SchemaDetail/Schema.jsx b/client/src/containers/Schema/SchemaDetail/Schema.jsx index c18be7afcb..aff67f92ac 100644 --- a/client/src/containers/Schema/SchemaDetail/Schema.jsx +++ b/client/src/containers/Schema/SchemaDetail/Schema.jsx @@ -2,6 +2,7 @@ import React from 'react'; import Header from '../../Header'; import SchemaVersions from './SchemaVersions'; import SchemaUpdate from './SchemaUpdate'; +import SchemaFields from './SchemaFields'; import endpoints from '../../../utils/endpoints'; import { getSelectedTab } from '../../../utils/functions'; import { Link } from 'react-router-dom'; @@ -18,7 +19,7 @@ class Schema extends Root { roles: JSON.parse(sessionStorage.getItem('roles')) }; - tabs = ['update', 'versions']; + tabs = ['update', 'versions', 'fields']; componentDidMount() { this.getSchemaVersions(); @@ -46,7 +47,7 @@ class Schema extends Root { { schemaVersions: schemas.data, totalVersions: schemas.data.length, - selectedTab: tabSelected === 'versions' ? tabSelected : 'update' + selectedTab: tabSelected }, () => { this.props.router.navigate( @@ -74,6 +75,8 @@ class Schema extends Root { schemaId={schemaId} /> ); + case 'fields': + return ; case 'versions': return ( @@ -109,6 +112,14 @@ class Schema extends Root { Versions {totalVersions} +
  • + + Fields + +
  • diff --git a/client/src/containers/Schema/SchemaDetail/SchemaFields/SchemaFields.jsx b/client/src/containers/Schema/SchemaDetail/SchemaFields/SchemaFields.jsx new file mode 100644 index 0000000000..45c081abf0 --- /dev/null +++ b/client/src/containers/Schema/SchemaDetail/SchemaFields/SchemaFields.jsx @@ -0,0 +1,142 @@ +import React, { useMemo, useState } from 'react'; +import PropTypes from 'prop-types'; +import { extractAvroFields, MAX_FIELDS } from '../../../../utils/avroFields'; + +const latestVersion = schemas => + schemas.reduce((latest, schema) => (schema.version > latest ? schema.version : latest), 0); + +const recordDoc = definition => { + try { + const parsed = JSON.parse(definition); + return parsed && typeof parsed === 'object' && !Array.isArray(parsed) ? parsed.doc : undefined; + } catch (e) { + return undefined; + } +}; + +const SchemaFields = ({ schemas }) => { + const versions = schemas || []; + const [selectedVersion, setSelectedVersion] = useState(() => latestVersion(versions)); + const [filter, setFilter] = useState(''); + + const schema = + versions.find(version => version.version === selectedVersion) ?? + versions.find(version => version.version === latestVersion(versions)); + // A version the server could not parse comes back without schema and schemaType. + const failed = schema && (schema.exception || !schema.schema); + const isAvro = schema && !failed && (!schema.schemaType || schema.schemaType === 'AVRO'); + const fields = useMemo(() => (isAvro ? extractAvroFields(schema.schema) : []), [isAvro, schema]); + + if (!schema) { + return null; + } + + const needle = filter.trim().toLowerCase(); + const visibleFields = needle + ? fields.filter( + field => + field.path.toLowerCase().includes(needle) || field.doc.toLowerCase().includes(needle) + ) + : fields; + const documentedCount = fields.filter(field => field.doc).length; + const description = isAvro ? recordDoc(schema.schema) : undefined; + + return ( +
    +
    + + + {isAvro && ( + <> + setFilter(event.target.value)} + /> + + {documentedCount} / {fields.length} fields documented + + + )} +
    + + {failed && ( +
    + This version could not be parsed{schema.exception ? `: ${schema.exception}` : '.'} +
    + )} + + {!failed && !isAvro && ( +
    + Field descriptions are only available for Avro schemas. +
    + )} + + {description &&

    {description}

    } + + {isAvro && fields.length >= MAX_FIELDS && ( +
    + Only the first {MAX_FIELDS} fields are listed. +
    + )} + + {isAvro && ( +
    + + + + + + + + + + {visibleFields.map((field, index) => ( + + + + + + ))} + +
    FieldTypeDescription
    + {field.path} + + {field.type} + {field.doc}
    +
    + )} +
    + ); +}; + +SchemaFields.propTypes = { + schemas: PropTypes.arrayOf( + PropTypes.shape({ + id: PropTypes.number, + version: PropTypes.number, + schemaType: PropTypes.string, + schema: PropTypes.string + }) + ) +}; + +export default SchemaFields; diff --git a/client/src/containers/Schema/SchemaDetail/SchemaFields/SchemaFields.test.jsx b/client/src/containers/Schema/SchemaDetail/SchemaFields/SchemaFields.test.jsx new file mode 100644 index 0000000000..22fd7367df --- /dev/null +++ b/client/src/containers/Schema/SchemaDetail/SchemaFields/SchemaFields.test.jsx @@ -0,0 +1,94 @@ +import React from 'react'; +import { afterEach, describe, expect, it } from 'vitest'; +import { cleanup, fireEvent, render, screen } from '@testing-library/react'; +import SchemaFields from './SchemaFields'; + +const v1 = { + id: 10, + version: 1, + schemaType: 'AVRO', + schema: JSON.stringify({ + type: 'record', + name: 'User', + fields: [{ name: 'id', type: 'long', doc: 'Old id' }] + }) +}; + +const v2 = { + id: 11, + version: 2, + schemaType: 'AVRO', + schema: JSON.stringify({ + type: 'record', + name: 'User', + doc: 'A registered user.', + fields: [ + { name: 'id', type: 'long', doc: 'Internal user id' }, + { name: 'nickname', type: ['null', 'string'], default: null } + ] + }) +}; + +describe('SchemaFields', () => { + afterEach(() => cleanup()); + + it('shows every field of the latest version by default', () => { + render(); + + expect(screen.getByText('A registered user.')).toBeTruthy(); + expect(screen.getByText('Internal user id')).toBeTruthy(); + expect(screen.getByText('nickname')).toBeTruthy(); + expect(screen.getByText('null | string')).toBeTruthy(); + expect(screen.getByText('1 / 2 fields documented')).toBeTruthy(); + expect(screen.queryByText('Old id')).toBeNull(); + }); + + it('switches to another version', () => { + render(); + + fireEvent.change(screen.getByLabelText('Version'), { target: { value: '1' } }); + + expect(screen.getByText('Old id')).toBeTruthy(); + expect(screen.queryByText('nickname')).toBeNull(); + }); + + it('filters by field path or description', () => { + render(); + + fireEvent.change(screen.getByLabelText('Filter fields'), { target: { value: 'internal' } }); + + expect(screen.getByText('Internal user id')).toBeTruthy(); + expect(screen.queryByText('nickname')).toBeNull(); + }); + + it('explains that only Avro schemas are supported', () => { + render( + + ); + + expect( + screen.getByText('Field descriptions are only available for Avro schemas.') + ).toBeTruthy(); + }); + + it('reports a version the server could not parse', () => { + render( + + ); + + expect(screen.getByText('This version could not be parsed: Invalid schema')).toBeTruthy(); + expect(screen.queryByText('Field')).toBeNull(); + }); + + it('renders nothing until the versions are loaded', () => { + const { container } = render(); + + expect(container.innerHTML).toBe(''); + }); +}); diff --git a/client/src/containers/Schema/SchemaDetail/SchemaFields/index.jsx b/client/src/containers/Schema/SchemaDetail/SchemaFields/index.jsx new file mode 100644 index 0000000000..1c8072e0f4 --- /dev/null +++ b/client/src/containers/Schema/SchemaDetail/SchemaFields/index.jsx @@ -0,0 +1,3 @@ +import SchemaFields from './SchemaFields'; + +export default SchemaFields; diff --git a/client/src/utils/avroFields.js b/client/src/utils/avroFields.js new file mode 100644 index 0000000000..e35cc21de2 --- /dev/null +++ b/client/src/utils/avroFields.js @@ -0,0 +1,179 @@ +const PRIMITIVES = new Set([ + 'null', + 'boolean', + 'int', + 'long', + 'float', + 'double', + 'bytes', + 'string' +]); +const NAMED = new Set(['record', 'error', 'enum', 'fixed']); + +const fullName = (name, namespace) => { + if (name.includes('.') || !namespace) { + return name; + } + return `${namespace}.${name}`; +}; + +const shortName = name => name.substring(name.lastIndexOf('.') + 1); + +const namespaceOf = (schema, enclosing) => { + if (schema.name && schema.name.includes('.')) { + return schema.name.substring(0, schema.name.lastIndexOf('.')); + } + return schema.namespace ?? enclosing; +}; + +// Each named type keeps the namespace it was defined in: short names inside it resolve against +// that namespace, not against the namespace of the field that references it. +const registerNamedTypes = (schema, namespace, registry) => { + if (Array.isArray(schema)) { + schema.forEach(branch => registerNamedTypes(branch, namespace, registry)); + return; + } + if (!schema || typeof schema !== 'object') { + return; + } + + if (NAMED.has(schema.type) && schema.name) { + const ns = namespaceOf(schema, namespace); + const name = fullName(schema.name, ns); + if (registry.has(name)) { + return; + } + registry.set(name, { schema, namespace: ns, name }); + (schema.fields || []).forEach(field => registerNamedTypes(field.type, ns, registry)); + return; + } + + if (schema.type === 'array') { + registerNamedTypes(schema.items, namespace, registry); + } else if (schema.type === 'map') { + registerNamedTypes(schema.values, namespace, registry); + } +}; + +const resolve = (schema, namespace, registry) => { + if (typeof schema === 'string') { + if (PRIMITIVES.has(schema)) { + return { schema, namespace }; + } + return ( + registry.get(fullName(schema, namespace)) ?? + registry.get(schema) ?? { schema, namespace, name: schema } + ); + } + if (schema && typeof schema === 'object' && NAMED.has(schema.type) && schema.name) { + const ns = namespaceOf(schema, namespace); + return { schema, namespace: ns, name: fullName(schema.name, ns) }; + } + return { schema, namespace }; +}; + +const isRecord = schema => + !!schema && typeof schema === 'object' && (schema.type === 'record' || schema.type === 'error'); + +const typeLabel = (schema, namespace, registry) => { + if (Array.isArray(schema)) { + return schema.map(branch => typeLabel(branch, namespace, registry)).join(' | '); + } + if (typeof schema === 'string') { + return PRIMITIVES.has(schema) ? schema : shortName(schema); + } + if (!schema || typeof schema !== 'object') { + return String(schema); + } + const logical = schema.logicalType ? ` (${schema.logicalType})` : ''; + if (NAMED.has(schema.type)) { + return `${shortName(schema.name)}${logical}`; + } + if (schema.type === 'array') { + return `array<${typeLabel(schema.items, namespace, registry)}>`; + } + if (schema.type === 'map') { + return `map<${typeLabel(schema.values, namespace, registry)}>`; + } + return `${typeLabel(schema.type, namespace, registry)}${logical}`; +}; + +/** Upper bound on listed fields: types shared at every level would otherwise grow exponentially. */ +export const MAX_FIELDS = 5000; + +const collect = (schema, namespace, path, registry, visiting, result) => { + if (result.length >= MAX_FIELDS) { + return; + } + + if (Array.isArray(schema)) { + const records = schema.filter(branch => isRecord(resolve(branch, namespace, registry).schema)); + schema.forEach(branch => { + const { schema: resolved } = resolve(branch, namespace, registry); + // With several record branches, the same field name can exist in more than one of them. + const branchPath = + records.length > 1 && isRecord(resolved) ? `${path}<${shortName(resolved.name)}>` : path; + collect(branch, namespace, branchPath, registry, visiting, result); + }); + return; + } + + const { schema: resolved, namespace: ns, name } = resolve(schema, namespace, registry); + if (!resolved || typeof resolved !== 'object') { + return; + } + + if (resolved.type === 'array') { + collect(resolved.items, ns, `${path}[]`, registry, visiting, result); + return; + } + if (resolved.type === 'map') { + collect(resolved.values, ns, `${path}{}`, registry, visiting, result); + return; + } + if (!isRecord(resolved)) { + return; + } + + // Recursive types (e.g. a linked list) would otherwise expand forever. + if (visiting.has(name)) { + return; + } + visiting.add(name); + + for (const field of resolved.fields || []) { + if (result.length >= MAX_FIELDS) { + break; + } + const fieldPath = path ? `${path}.${field.name}` : field.name; + result.push({ + path: fieldPath, + type: typeLabel(field.type, ns, registry), + doc: field.doc ?? '' + }); + collect(field.type, ns, fieldPath, registry, visiting, result); + } + + visiting.delete(name); +}; + +/** + * Lists every field of an Avro schema, nested ones included, as `{ path, type, doc }` in + * declaration order, up to MAX_FIELDS entries. Array items are addressed with `[]`, map values + * with `{}`, and a record branch of a union holding several records with ``. + */ +export const extractAvroFields = schemaDefinition => { + let schema; + try { + schema = typeof schemaDefinition === 'string' ? JSON.parse(schemaDefinition) : schemaDefinition; + } catch (e) { + return []; + } + + const registry = new Map(); + registerNamedTypes(schema, undefined, registry); + + const result = []; + collect(schema, undefined, '', registry, new Set(), result); + return result; +}; diff --git a/client/src/utils/avroFields.test.js b/client/src/utils/avroFields.test.js new file mode 100644 index 0000000000..1a3d2951fb --- /dev/null +++ b/client/src/utils/avroFields.test.js @@ -0,0 +1,232 @@ +import { describe, expect, it } from 'vitest'; +import { extractAvroFields, MAX_FIELDS } from './avroFields'; + +describe('extractAvroFields', () => { + it('lists every top-level field with its type and doc', () => { + const schema = { + type: 'record', + name: 'User', + fields: [ + { name: 'id', type: 'long', doc: 'Internal user id' }, + { name: 'email', type: ['null', 'string'], doc: 'Contact address' }, + { + name: 'createdAt', + type: { type: 'long', logicalType: 'timestamp-millis' }, + doc: 'Creation time' + }, + { name: 'undocumented', type: 'string' } + ] + }; + + expect(extractAvroFields(JSON.stringify(schema))).toEqual([ + { path: 'id', type: 'long', doc: 'Internal user id' }, + { path: 'email', type: 'null | string', doc: 'Contact address' }, + { path: 'createdAt', type: 'long (timestamp-millis)', doc: 'Creation time' }, + { path: 'undocumented', type: 'string', doc: '' } + ]); + }); + + it('walks nested records, unions, arrays and maps', () => { + const schema = { + type: 'record', + name: 'Order', + namespace: 'com.example', + fields: [ + { + name: 'customer', + type: [ + 'null', + { + type: 'record', + name: 'Customer', + fields: [{ name: 'name', type: 'string', doc: 'Display name' }] + } + ], + doc: 'Who placed the order' + }, + { + name: 'lines', + type: { + type: 'array', + items: { + type: 'record', + name: 'Line', + fields: [{ name: 'sku', type: 'string', doc: 'Product code' }] + } + } + }, + { + name: 'attributes', + type: { + type: 'map', + values: { + type: 'record', + name: 'Attr', + fields: [{ name: 'v', type: 'int', doc: 'Value' }] + } + } + }, + { name: 'status', type: { type: 'enum', name: 'Status', symbols: ['NEW', 'PAID'] } } + ] + }; + + expect(extractAvroFields(schema)).toEqual([ + { path: 'customer', type: 'null | Customer', doc: 'Who placed the order' }, + { path: 'customer.name', type: 'string', doc: 'Display name' }, + { path: 'lines', type: 'array', doc: '' }, + { path: 'lines[].sku', type: 'string', doc: 'Product code' }, + { path: 'attributes', type: 'map', doc: '' }, + { path: 'attributes{}.v', type: 'int', doc: 'Value' }, + { path: 'status', type: 'Status', doc: '' } + ]); + }); + + it('resolves named type references, including across namespaces', () => { + const schema = { + type: 'record', + name: 'Shipment', + namespace: 'com.example', + fields: [ + { + name: 'from', + type: { + type: 'record', + name: 'Address', + fields: [{ name: 'city', type: 'string', doc: 'City name' }] + } + }, + { name: 'to', type: 'Address' }, + { name: 'via', type: ['null', 'com.example.Address'] } + ] + }; + + expect(extractAvroFields(schema).map(entry => entry.path)).toEqual([ + 'from', + 'from.city', + 'to', + 'to.city', + 'via', + 'via.city' + ]); + }); + + it('stops on recursive types', () => { + const schema = { + type: 'record', + name: 'Node', + fields: [ + { name: 'label', type: 'string', doc: 'Label' }, + { name: 'next', type: ['null', 'Node'], doc: 'Next node' } + ] + }; + + expect(extractAvroFields(schema)).toEqual([ + { path: 'label', type: 'string', doc: 'Label' }, + { path: 'next', type: 'null | Node', doc: 'Next node' } + ]); + }); + + it('resolves short names inside a referenced type against its own namespace', () => { + const schema = { + type: 'record', + name: 'Root', + namespace: 'a', + fields: [ + { + name: 'inner', + type: { + type: 'record', + name: 'Inner', + fields: [ + { + name: 'leaf', + type: { type: 'record', name: 'Leaf', fields: [{ name: 'v', type: 'int' }] } + }, + { name: 'leaf2', type: 'Leaf' } + ] + } + }, + { + name: 'other', + type: { + type: 'record', + name: 'Other', + namespace: 'b', + fields: [{ name: 'ref', type: 'a.Inner' }] + } + } + ] + }; + + expect(extractAvroFields(schema).map(entry => entry.path)).toContain('other.ref.leaf2.v'); + }); + + it('tells apart the record branches of a union', () => { + const schema = { + type: 'record', + name: 'Payment', + fields: [ + { + name: 'method', + type: [ + 'null', + { + type: 'record', + name: 'Card', + fields: [{ name: 'id', type: 'string', doc: 'Card id' }] + }, + { type: 'record', name: 'Bank', fields: [{ name: 'id', type: 'long', doc: 'Bank id' }] } + ] + } + ] + }; + + expect(extractAvroFields(schema)).toEqual([ + { path: 'method', type: 'null | Card | Bank', doc: '' }, + { path: 'method.id', type: 'string', doc: 'Card id' }, + { path: 'method.id', type: 'long', doc: 'Bank id' } + ]); + }); + + it('shows logical types of named types', () => { + const schema = { + type: 'record', + name: 'Price', + fields: [ + { + name: 'amount', + type: { type: 'fixed', name: 'Amount', size: 8, logicalType: 'decimal', precision: 10 } + } + ] + }; + + expect(extractAvroFields(schema)[0].type).toBe('Amount (decimal)'); + }); + + it('stops listing at MAX_FIELDS when shared types blow up', () => { + // Every level references the next type twice, which doubles the paths at each level. + const levels = 20; + const types = []; + for (let i = levels; i >= 0; i--) { + types[i] = + i === levels + ? { type: 'record', name: `T${i}`, fields: [{ name: 'v', type: 'int' }] } + : { + type: 'record', + name: `T${i}`, + fields: [ + { name: 'a', type: types[i + 1] }, + { name: 'b', type: `T${i + 1}` } + ] + }; + } + + expect(extractAvroFields(types[0])).toHaveLength(MAX_FIELDS); + }); + + it('returns nothing for schemas it cannot read', () => { + expect(extractAvroFields('not json')).toEqual([]); + expect(extractAvroFields('"string"')).toEqual([]); + expect(extractAvroFields(undefined)).toEqual([]); + }); +});