diff --git a/client/src/containers/Schema/SchemaDetail/Schema.jsx b/client/src/containers/Schema/SchemaDetail/Schema.jsx index c18be7afc..aff67f92a 100644 --- a/client/src/containers/Schema/SchemaDetail/Schema.jsx +++ b/client/src/containers/Schema/SchemaDetail/Schema.jsx @@ -2,6 +2,7 @@ import React from 'react'; import Header from '../../Header'; import SchemaVersions from './SchemaVersions'; import SchemaUpdate from './SchemaUpdate'; +import SchemaFields from './SchemaFields'; import endpoints from '../../../utils/endpoints'; import { getSelectedTab } from '../../../utils/functions'; import { Link } from 'react-router-dom'; @@ -18,7 +19,7 @@ class Schema extends Root { roles: JSON.parse(sessionStorage.getItem('roles')) }; - tabs = ['update', 'versions']; + tabs = ['update', 'versions', 'fields']; componentDidMount() { this.getSchemaVersions(); @@ -46,7 +47,7 @@ class Schema extends Root { { schemaVersions: schemas.data, totalVersions: schemas.data.length, - selectedTab: tabSelected === 'versions' ? tabSelected : 'update' + selectedTab: tabSelected }, () => { this.props.router.navigate( @@ -74,6 +75,8 @@ class Schema extends Root { schemaId={schemaId} /> ); + case 'fields': + return ; case 'versions': return ( @@ -109,6 +112,14 @@ class Schema extends Root { Versions {totalVersions} +
  • + + Fields + +
  • diff --git a/client/src/containers/Schema/SchemaDetail/SchemaFields/SchemaFields.jsx b/client/src/containers/Schema/SchemaDetail/SchemaFields/SchemaFields.jsx new file mode 100644 index 000000000..45c081abf --- /dev/null +++ b/client/src/containers/Schema/SchemaDetail/SchemaFields/SchemaFields.jsx @@ -0,0 +1,142 @@ +import React, { useMemo, useState } from 'react'; +import PropTypes from 'prop-types'; +import { extractAvroFields, MAX_FIELDS } from '../../../../utils/avroFields'; + +const latestVersion = schemas => + schemas.reduce((latest, schema) => (schema.version > latest ? schema.version : latest), 0); + +const recordDoc = definition => { + try { + const parsed = JSON.parse(definition); + return parsed && typeof parsed === 'object' && !Array.isArray(parsed) ? parsed.doc : undefined; + } catch (e) { + return undefined; + } +}; + +const SchemaFields = ({ schemas }) => { + const versions = schemas || []; + const [selectedVersion, setSelectedVersion] = useState(() => latestVersion(versions)); + const [filter, setFilter] = useState(''); + + const schema = + versions.find(version => version.version === selectedVersion) ?? + versions.find(version => version.version === latestVersion(versions)); + // A version the server could not parse comes back without schema and schemaType. + const failed = schema && (schema.exception || !schema.schema); + const isAvro = schema && !failed && (!schema.schemaType || schema.schemaType === 'AVRO'); + const fields = useMemo(() => (isAvro ? extractAvroFields(schema.schema) : []), [isAvro, schema]); + + if (!schema) { + return null; + } + + const needle = filter.trim().toLowerCase(); + const visibleFields = needle + ? fields.filter( + field => + field.path.toLowerCase().includes(needle) || field.doc.toLowerCase().includes(needle) + ) + : fields; + const documentedCount = fields.filter(field => field.doc).length; + const description = isAvro ? recordDoc(schema.schema) : undefined; + + return ( +
    +
    + + + {isAvro && ( + <> + setFilter(event.target.value)} + /> + + {documentedCount} / {fields.length} fields documented + + + )} +
    + + {failed && ( +
    + This version could not be parsed{schema.exception ? `: ${schema.exception}` : '.'} +
    + )} + + {!failed && !isAvro && ( +
    + Field descriptions are only available for Avro schemas. +
    + )} + + {description &&

    {description}

    } + + {isAvro && fields.length >= MAX_FIELDS && ( +
    + Only the first {MAX_FIELDS} fields are listed. +
    + )} + + {isAvro && ( +
    + + + + + + + + + + {visibleFields.map((field, index) => ( + + + + + + ))} + +
    FieldTypeDescription
    + {field.path} + + {field.type} + {field.doc}
    +
    + )} +
    + ); +}; + +SchemaFields.propTypes = { + schemas: PropTypes.arrayOf( + PropTypes.shape({ + id: PropTypes.number, + version: PropTypes.number, + schemaType: PropTypes.string, + schema: PropTypes.string + }) + ) +}; + +export default SchemaFields; diff --git a/client/src/containers/Schema/SchemaDetail/SchemaFields/SchemaFields.test.jsx b/client/src/containers/Schema/SchemaDetail/SchemaFields/SchemaFields.test.jsx new file mode 100644 index 000000000..22fd7367d --- /dev/null +++ b/client/src/containers/Schema/SchemaDetail/SchemaFields/SchemaFields.test.jsx @@ -0,0 +1,94 @@ +import React from 'react'; +import { afterEach, describe, expect, it } from 'vitest'; +import { cleanup, fireEvent, render, screen } from '@testing-library/react'; +import SchemaFields from './SchemaFields'; + +const v1 = { + id: 10, + version: 1, + schemaType: 'AVRO', + schema: JSON.stringify({ + type: 'record', + name: 'User', + fields: [{ name: 'id', type: 'long', doc: 'Old id' }] + }) +}; + +const v2 = { + id: 11, + version: 2, + schemaType: 'AVRO', + schema: JSON.stringify({ + type: 'record', + name: 'User', + doc: 'A registered user.', + fields: [ + { name: 'id', type: 'long', doc: 'Internal user id' }, + { name: 'nickname', type: ['null', 'string'], default: null } + ] + }) +}; + +describe('SchemaFields', () => { + afterEach(() => cleanup()); + + it('shows every field of the latest version by default', () => { + render(); + + expect(screen.getByText('A registered user.')).toBeTruthy(); + expect(screen.getByText('Internal user id')).toBeTruthy(); + expect(screen.getByText('nickname')).toBeTruthy(); + expect(screen.getByText('null | string')).toBeTruthy(); + expect(screen.getByText('1 / 2 fields documented')).toBeTruthy(); + expect(screen.queryByText('Old id')).toBeNull(); + }); + + it('switches to another version', () => { + render(); + + fireEvent.change(screen.getByLabelText('Version'), { target: { value: '1' } }); + + expect(screen.getByText('Old id')).toBeTruthy(); + expect(screen.queryByText('nickname')).toBeNull(); + }); + + it('filters by field path or description', () => { + render(); + + fireEvent.change(screen.getByLabelText('Filter fields'), { target: { value: 'internal' } }); + + expect(screen.getByText('Internal user id')).toBeTruthy(); + expect(screen.queryByText('nickname')).toBeNull(); + }); + + it('explains that only Avro schemas are supported', () => { + render( + + ); + + expect( + screen.getByText('Field descriptions are only available for Avro schemas.') + ).toBeTruthy(); + }); + + it('reports a version the server could not parse', () => { + render( + + ); + + expect(screen.getByText('This version could not be parsed: Invalid schema')).toBeTruthy(); + expect(screen.queryByText('Field')).toBeNull(); + }); + + it('renders nothing until the versions are loaded', () => { + const { container } = render(); + + expect(container.innerHTML).toBe(''); + }); +}); diff --git a/client/src/containers/Schema/SchemaDetail/SchemaFields/index.jsx b/client/src/containers/Schema/SchemaDetail/SchemaFields/index.jsx new file mode 100644 index 000000000..1c8072e0f --- /dev/null +++ b/client/src/containers/Schema/SchemaDetail/SchemaFields/index.jsx @@ -0,0 +1,3 @@ +import SchemaFields from './SchemaFields'; + +export default SchemaFields; diff --git a/client/src/utils/avroFields.js b/client/src/utils/avroFields.js new file mode 100644 index 000000000..e35cc21de --- /dev/null +++ b/client/src/utils/avroFields.js @@ -0,0 +1,179 @@ +const PRIMITIVES = new Set([ + 'null', + 'boolean', + 'int', + 'long', + 'float', + 'double', + 'bytes', + 'string' +]); +const NAMED = new Set(['record', 'error', 'enum', 'fixed']); + +const fullName = (name, namespace) => { + if (name.includes('.') || !namespace) { + return name; + } + return `${namespace}.${name}`; +}; + +const shortName = name => name.substring(name.lastIndexOf('.') + 1); + +const namespaceOf = (schema, enclosing) => { + if (schema.name && schema.name.includes('.')) { + return schema.name.substring(0, schema.name.lastIndexOf('.')); + } + return schema.namespace ?? enclosing; +}; + +// Each named type keeps the namespace it was defined in: short names inside it resolve against +// that namespace, not against the namespace of the field that references it. +const registerNamedTypes = (schema, namespace, registry) => { + if (Array.isArray(schema)) { + schema.forEach(branch => registerNamedTypes(branch, namespace, registry)); + return; + } + if (!schema || typeof schema !== 'object') { + return; + } + + if (NAMED.has(schema.type) && schema.name) { + const ns = namespaceOf(schema, namespace); + const name = fullName(schema.name, ns); + if (registry.has(name)) { + return; + } + registry.set(name, { schema, namespace: ns, name }); + (schema.fields || []).forEach(field => registerNamedTypes(field.type, ns, registry)); + return; + } + + if (schema.type === 'array') { + registerNamedTypes(schema.items, namespace, registry); + } else if (schema.type === 'map') { + registerNamedTypes(schema.values, namespace, registry); + } +}; + +const resolve = (schema, namespace, registry) => { + if (typeof schema === 'string') { + if (PRIMITIVES.has(schema)) { + return { schema, namespace }; + } + return ( + registry.get(fullName(schema, namespace)) ?? + registry.get(schema) ?? { schema, namespace, name: schema } + ); + } + if (schema && typeof schema === 'object' && NAMED.has(schema.type) && schema.name) { + const ns = namespaceOf(schema, namespace); + return { schema, namespace: ns, name: fullName(schema.name, ns) }; + } + return { schema, namespace }; +}; + +const isRecord = schema => + !!schema && typeof schema === 'object' && (schema.type === 'record' || schema.type === 'error'); + +const typeLabel = (schema, namespace, registry) => { + if (Array.isArray(schema)) { + return schema.map(branch => typeLabel(branch, namespace, registry)).join(' | '); + } + if (typeof schema === 'string') { + return PRIMITIVES.has(schema) ? schema : shortName(schema); + } + if (!schema || typeof schema !== 'object') { + return String(schema); + } + const logical = schema.logicalType ? ` (${schema.logicalType})` : ''; + if (NAMED.has(schema.type)) { + return `${shortName(schema.name)}${logical}`; + } + if (schema.type === 'array') { + return `array<${typeLabel(schema.items, namespace, registry)}>`; + } + if (schema.type === 'map') { + return `map<${typeLabel(schema.values, namespace, registry)}>`; + } + return `${typeLabel(schema.type, namespace, registry)}${logical}`; +}; + +/** Upper bound on listed fields: types shared at every level would otherwise grow exponentially. */ +export const MAX_FIELDS = 5000; + +const collect = (schema, namespace, path, registry, visiting, result) => { + if (result.length >= MAX_FIELDS) { + return; + } + + if (Array.isArray(schema)) { + const records = schema.filter(branch => isRecord(resolve(branch, namespace, registry).schema)); + schema.forEach(branch => { + const { schema: resolved } = resolve(branch, namespace, registry); + // With several record branches, the same field name can exist in more than one of them. + const branchPath = + records.length > 1 && isRecord(resolved) ? `${path}<${shortName(resolved.name)}>` : path; + collect(branch, namespace, branchPath, registry, visiting, result); + }); + return; + } + + const { schema: resolved, namespace: ns, name } = resolve(schema, namespace, registry); + if (!resolved || typeof resolved !== 'object') { + return; + } + + if (resolved.type === 'array') { + collect(resolved.items, ns, `${path}[]`, registry, visiting, result); + return; + } + if (resolved.type === 'map') { + collect(resolved.values, ns, `${path}{}`, registry, visiting, result); + return; + } + if (!isRecord(resolved)) { + return; + } + + // Recursive types (e.g. a linked list) would otherwise expand forever. + if (visiting.has(name)) { + return; + } + visiting.add(name); + + for (const field of resolved.fields || []) { + if (result.length >= MAX_FIELDS) { + break; + } + const fieldPath = path ? `${path}.${field.name}` : field.name; + result.push({ + path: fieldPath, + type: typeLabel(field.type, ns, registry), + doc: field.doc ?? '' + }); + collect(field.type, ns, fieldPath, registry, visiting, result); + } + + visiting.delete(name); +}; + +/** + * Lists every field of an Avro schema, nested ones included, as `{ path, type, doc }` in + * declaration order, up to MAX_FIELDS entries. Array items are addressed with `[]`, map values + * with `{}`, and a record branch of a union holding several records with ``. + */ +export const extractAvroFields = schemaDefinition => { + let schema; + try { + schema = typeof schemaDefinition === 'string' ? JSON.parse(schemaDefinition) : schemaDefinition; + } catch (e) { + return []; + } + + const registry = new Map(); + registerNamedTypes(schema, undefined, registry); + + const result = []; + collect(schema, undefined, '', registry, new Set(), result); + return result; +}; diff --git a/client/src/utils/avroFields.test.js b/client/src/utils/avroFields.test.js new file mode 100644 index 000000000..1a3d2951f --- /dev/null +++ b/client/src/utils/avroFields.test.js @@ -0,0 +1,232 @@ +import { describe, expect, it } from 'vitest'; +import { extractAvroFields, MAX_FIELDS } from './avroFields'; + +describe('extractAvroFields', () => { + it('lists every top-level field with its type and doc', () => { + const schema = { + type: 'record', + name: 'User', + fields: [ + { name: 'id', type: 'long', doc: 'Internal user id' }, + { name: 'email', type: ['null', 'string'], doc: 'Contact address' }, + { + name: 'createdAt', + type: { type: 'long', logicalType: 'timestamp-millis' }, + doc: 'Creation time' + }, + { name: 'undocumented', type: 'string' } + ] + }; + + expect(extractAvroFields(JSON.stringify(schema))).toEqual([ + { path: 'id', type: 'long', doc: 'Internal user id' }, + { path: 'email', type: 'null | string', doc: 'Contact address' }, + { path: 'createdAt', type: 'long (timestamp-millis)', doc: 'Creation time' }, + { path: 'undocumented', type: 'string', doc: '' } + ]); + }); + + it('walks nested records, unions, arrays and maps', () => { + const schema = { + type: 'record', + name: 'Order', + namespace: 'com.example', + fields: [ + { + name: 'customer', + type: [ + 'null', + { + type: 'record', + name: 'Customer', + fields: [{ name: 'name', type: 'string', doc: 'Display name' }] + } + ], + doc: 'Who placed the order' + }, + { + name: 'lines', + type: { + type: 'array', + items: { + type: 'record', + name: 'Line', + fields: [{ name: 'sku', type: 'string', doc: 'Product code' }] + } + } + }, + { + name: 'attributes', + type: { + type: 'map', + values: { + type: 'record', + name: 'Attr', + fields: [{ name: 'v', type: 'int', doc: 'Value' }] + } + } + }, + { name: 'status', type: { type: 'enum', name: 'Status', symbols: ['NEW', 'PAID'] } } + ] + }; + + expect(extractAvroFields(schema)).toEqual([ + { path: 'customer', type: 'null | Customer', doc: 'Who placed the order' }, + { path: 'customer.name', type: 'string', doc: 'Display name' }, + { path: 'lines', type: 'array', doc: '' }, + { path: 'lines[].sku', type: 'string', doc: 'Product code' }, + { path: 'attributes', type: 'map', doc: '' }, + { path: 'attributes{}.v', type: 'int', doc: 'Value' }, + { path: 'status', type: 'Status', doc: '' } + ]); + }); + + it('resolves named type references, including across namespaces', () => { + const schema = { + type: 'record', + name: 'Shipment', + namespace: 'com.example', + fields: [ + { + name: 'from', + type: { + type: 'record', + name: 'Address', + fields: [{ name: 'city', type: 'string', doc: 'City name' }] + } + }, + { name: 'to', type: 'Address' }, + { name: 'via', type: ['null', 'com.example.Address'] } + ] + }; + + expect(extractAvroFields(schema).map(entry => entry.path)).toEqual([ + 'from', + 'from.city', + 'to', + 'to.city', + 'via', + 'via.city' + ]); + }); + + it('stops on recursive types', () => { + const schema = { + type: 'record', + name: 'Node', + fields: [ + { name: 'label', type: 'string', doc: 'Label' }, + { name: 'next', type: ['null', 'Node'], doc: 'Next node' } + ] + }; + + expect(extractAvroFields(schema)).toEqual([ + { path: 'label', type: 'string', doc: 'Label' }, + { path: 'next', type: 'null | Node', doc: 'Next node' } + ]); + }); + + it('resolves short names inside a referenced type against its own namespace', () => { + const schema = { + type: 'record', + name: 'Root', + namespace: 'a', + fields: [ + { + name: 'inner', + type: { + type: 'record', + name: 'Inner', + fields: [ + { + name: 'leaf', + type: { type: 'record', name: 'Leaf', fields: [{ name: 'v', type: 'int' }] } + }, + { name: 'leaf2', type: 'Leaf' } + ] + } + }, + { + name: 'other', + type: { + type: 'record', + name: 'Other', + namespace: 'b', + fields: [{ name: 'ref', type: 'a.Inner' }] + } + } + ] + }; + + expect(extractAvroFields(schema).map(entry => entry.path)).toContain('other.ref.leaf2.v'); + }); + + it('tells apart the record branches of a union', () => { + const schema = { + type: 'record', + name: 'Payment', + fields: [ + { + name: 'method', + type: [ + 'null', + { + type: 'record', + name: 'Card', + fields: [{ name: 'id', type: 'string', doc: 'Card id' }] + }, + { type: 'record', name: 'Bank', fields: [{ name: 'id', type: 'long', doc: 'Bank id' }] } + ] + } + ] + }; + + expect(extractAvroFields(schema)).toEqual([ + { path: 'method', type: 'null | Card | Bank', doc: '' }, + { path: 'method.id', type: 'string', doc: 'Card id' }, + { path: 'method.id', type: 'long', doc: 'Bank id' } + ]); + }); + + it('shows logical types of named types', () => { + const schema = { + type: 'record', + name: 'Price', + fields: [ + { + name: 'amount', + type: { type: 'fixed', name: 'Amount', size: 8, logicalType: 'decimal', precision: 10 } + } + ] + }; + + expect(extractAvroFields(schema)[0].type).toBe('Amount (decimal)'); + }); + + it('stops listing at MAX_FIELDS when shared types blow up', () => { + // Every level references the next type twice, which doubles the paths at each level. + const levels = 20; + const types = []; + for (let i = levels; i >= 0; i--) { + types[i] = + i === levels + ? { type: 'record', name: `T${i}`, fields: [{ name: 'v', type: 'int' }] } + : { + type: 'record', + name: `T${i}`, + fields: [ + { name: 'a', type: types[i + 1] }, + { name: 'b', type: `T${i + 1}` } + ] + }; + } + + expect(extractAvroFields(types[0])).toHaveLength(MAX_FIELDS); + }); + + it('returns nothing for schemas it cannot read', () => { + expect(extractAvroFields('not json')).toEqual([]); + expect(extractAvroFields('"string"')).toEqual([]); + expect(extractAvroFields(undefined)).toEqual([]); + }); +});