diff --git a/client/src/containers/Schema/SchemaDetail/Schema.jsx b/client/src/containers/Schema/SchemaDetail/Schema.jsx
index c18be7afc..aff67f92a 100644
--- a/client/src/containers/Schema/SchemaDetail/Schema.jsx
+++ b/client/src/containers/Schema/SchemaDetail/Schema.jsx
@@ -2,6 +2,7 @@ import React from 'react';
import Header from '../../Header';
import SchemaVersions from './SchemaVersions';
import SchemaUpdate from './SchemaUpdate';
+import SchemaFields from './SchemaFields';
import endpoints from '../../../utils/endpoints';
import { getSelectedTab } from '../../../utils/functions';
import { Link } from 'react-router-dom';
@@ -18,7 +19,7 @@ class Schema extends Root {
roles: JSON.parse(sessionStorage.getItem('roles'))
};
- tabs = ['update', 'versions'];
+ tabs = ['update', 'versions', 'fields'];
componentDidMount() {
this.getSchemaVersions();
@@ -46,7 +47,7 @@ class Schema extends Root {
{
schemaVersions: schemas.data,
totalVersions: schemas.data.length,
- selectedTab: tabSelected === 'versions' ? tabSelected : 'update'
+ selectedTab: tabSelected
},
() => {
this.props.router.navigate(
@@ -74,6 +75,8 @@ class Schema extends Root {
schemaId={schemaId}
/>
);
+ case 'fields':
+ return ;
case 'versions':
return (
@@ -109,6 +112,14 @@ class Schema extends Root {
Versions {totalVersions}
+
diff --git a/client/src/containers/Schema/SchemaDetail/SchemaFields/SchemaFields.jsx b/client/src/containers/Schema/SchemaDetail/SchemaFields/SchemaFields.jsx
new file mode 100644
index 000000000..45c081abf
--- /dev/null
+++ b/client/src/containers/Schema/SchemaDetail/SchemaFields/SchemaFields.jsx
@@ -0,0 +1,142 @@
+import React, { useMemo, useState } from 'react';
+import PropTypes from 'prop-types';
+import { extractAvroFields, MAX_FIELDS } from '../../../../utils/avroFields';
+
+const latestVersion = schemas =>
+ schemas.reduce((latest, schema) => (schema.version > latest ? schema.version : latest), 0);
+
+const recordDoc = definition => {
+ try {
+ const parsed = JSON.parse(definition);
+ return parsed && typeof parsed === 'object' && !Array.isArray(parsed) ? parsed.doc : undefined;
+ } catch (e) {
+ return undefined;
+ }
+};
+
+const SchemaFields = ({ schemas }) => {
+ const versions = schemas || [];
+ const [selectedVersion, setSelectedVersion] = useState(() => latestVersion(versions));
+ const [filter, setFilter] = useState('');
+
+ const schema =
+ versions.find(version => version.version === selectedVersion) ??
+ versions.find(version => version.version === latestVersion(versions));
+ // A version the server could not parse comes back without schema and schemaType.
+ const failed = schema && (schema.exception || !schema.schema);
+ const isAvro = schema && !failed && (!schema.schemaType || schema.schemaType === 'AVRO');
+ const fields = useMemo(() => (isAvro ? extractAvroFields(schema.schema) : []), [isAvro, schema]);
+
+ if (!schema) {
+ return null;
+ }
+
+ const needle = filter.trim().toLowerCase();
+ const visibleFields = needle
+ ? fields.filter(
+ field =>
+ field.path.toLowerCase().includes(needle) || field.doc.toLowerCase().includes(needle)
+ )
+ : fields;
+ const documentedCount = fields.filter(field => field.doc).length;
+ const description = isAvro ? recordDoc(schema.schema) : undefined;
+
+ return (
+
+
+
+
+ {isAvro && (
+ <>
+ setFilter(event.target.value)}
+ />
+
+ {documentedCount} / {fields.length} fields documented
+
+ >
+ )}
+
+
+ {failed && (
+
+ This version could not be parsed{schema.exception ? `: ${schema.exception}` : '.'}
+
+ )}
+
+ {!failed && !isAvro && (
+
+ Field descriptions are only available for Avro schemas.
+
+ )}
+
+ {description &&
{description}
}
+
+ {isAvro && fields.length >= MAX_FIELDS && (
+
+ Only the first {MAX_FIELDS} fields are listed.
+
+ )}
+
+ {isAvro && (
+
+
+
+
+ | Field |
+ Type |
+ Description |
+
+
+
+ {visibleFields.map((field, index) => (
+
+
+ {field.path}
+ |
+
+ {field.type}
+ |
+ {field.doc} |
+
+ ))}
+
+
+
+ )}
+
+ );
+};
+
+SchemaFields.propTypes = {
+ schemas: PropTypes.arrayOf(
+ PropTypes.shape({
+ id: PropTypes.number,
+ version: PropTypes.number,
+ schemaType: PropTypes.string,
+ schema: PropTypes.string
+ })
+ )
+};
+
+export default SchemaFields;
diff --git a/client/src/containers/Schema/SchemaDetail/SchemaFields/SchemaFields.test.jsx b/client/src/containers/Schema/SchemaDetail/SchemaFields/SchemaFields.test.jsx
new file mode 100644
index 000000000..22fd7367d
--- /dev/null
+++ b/client/src/containers/Schema/SchemaDetail/SchemaFields/SchemaFields.test.jsx
@@ -0,0 +1,94 @@
+import React from 'react';
+import { afterEach, describe, expect, it } from 'vitest';
+import { cleanup, fireEvent, render, screen } from '@testing-library/react';
+import SchemaFields from './SchemaFields';
+
+const v1 = {
+ id: 10,
+ version: 1,
+ schemaType: 'AVRO',
+ schema: JSON.stringify({
+ type: 'record',
+ name: 'User',
+ fields: [{ name: 'id', type: 'long', doc: 'Old id' }]
+ })
+};
+
+const v2 = {
+ id: 11,
+ version: 2,
+ schemaType: 'AVRO',
+ schema: JSON.stringify({
+ type: 'record',
+ name: 'User',
+ doc: 'A registered user.',
+ fields: [
+ { name: 'id', type: 'long', doc: 'Internal user id' },
+ { name: 'nickname', type: ['null', 'string'], default: null }
+ ]
+ })
+};
+
+describe('SchemaFields', () => {
+ afterEach(() => cleanup());
+
+ it('shows every field of the latest version by default', () => {
+ render(
);
+
+ expect(screen.getByText('A registered user.')).toBeTruthy();
+ expect(screen.getByText('Internal user id')).toBeTruthy();
+ expect(screen.getByText('nickname')).toBeTruthy();
+ expect(screen.getByText('null | string')).toBeTruthy();
+ expect(screen.getByText('1 / 2 fields documented')).toBeTruthy();
+ expect(screen.queryByText('Old id')).toBeNull();
+ });
+
+ it('switches to another version', () => {
+ render(
);
+
+ fireEvent.change(screen.getByLabelText('Version'), { target: { value: '1' } });
+
+ expect(screen.getByText('Old id')).toBeTruthy();
+ expect(screen.queryByText('nickname')).toBeNull();
+ });
+
+ it('filters by field path or description', () => {
+ render(
);
+
+ fireEvent.change(screen.getByLabelText('Filter fields'), { target: { value: 'internal' } });
+
+ expect(screen.getByText('Internal user id')).toBeTruthy();
+ expect(screen.queryByText('nickname')).toBeNull();
+ });
+
+ it('explains that only Avro schemas are supported', () => {
+ render(
+
+ );
+
+ expect(
+ screen.getByText('Field descriptions are only available for Avro schemas.')
+ ).toBeTruthy();
+ });
+
+ it('reports a version the server could not parse', () => {
+ render(
+
+ );
+
+ expect(screen.getByText('This version could not be parsed: Invalid schema')).toBeTruthy();
+ expect(screen.queryByText('Field')).toBeNull();
+ });
+
+ it('renders nothing until the versions are loaded', () => {
+ const { container } = render(
);
+
+ expect(container.innerHTML).toBe('');
+ });
+});
diff --git a/client/src/containers/Schema/SchemaDetail/SchemaFields/index.jsx b/client/src/containers/Schema/SchemaDetail/SchemaFields/index.jsx
new file mode 100644
index 000000000..1c8072e0f
--- /dev/null
+++ b/client/src/containers/Schema/SchemaDetail/SchemaFields/index.jsx
@@ -0,0 +1,3 @@
+import SchemaFields from './SchemaFields';
+
+export default SchemaFields;
diff --git a/client/src/utils/avroFields.js b/client/src/utils/avroFields.js
new file mode 100644
index 000000000..e35cc21de
--- /dev/null
+++ b/client/src/utils/avroFields.js
@@ -0,0 +1,179 @@
+const PRIMITIVES = new Set([
+ 'null',
+ 'boolean',
+ 'int',
+ 'long',
+ 'float',
+ 'double',
+ 'bytes',
+ 'string'
+]);
+const NAMED = new Set(['record', 'error', 'enum', 'fixed']);
+
+const fullName = (name, namespace) => {
+ if (name.includes('.') || !namespace) {
+ return name;
+ }
+ return `${namespace}.${name}`;
+};
+
+const shortName = name => name.substring(name.lastIndexOf('.') + 1);
+
+const namespaceOf = (schema, enclosing) => {
+ if (schema.name && schema.name.includes('.')) {
+ return schema.name.substring(0, schema.name.lastIndexOf('.'));
+ }
+ return schema.namespace ?? enclosing;
+};
+
+// Each named type keeps the namespace it was defined in: short names inside it resolve against
+// that namespace, not against the namespace of the field that references it.
+const registerNamedTypes = (schema, namespace, registry) => {
+ if (Array.isArray(schema)) {
+ schema.forEach(branch => registerNamedTypes(branch, namespace, registry));
+ return;
+ }
+ if (!schema || typeof schema !== 'object') {
+ return;
+ }
+
+ if (NAMED.has(schema.type) && schema.name) {
+ const ns = namespaceOf(schema, namespace);
+ const name = fullName(schema.name, ns);
+ if (registry.has(name)) {
+ return;
+ }
+ registry.set(name, { schema, namespace: ns, name });
+ (schema.fields || []).forEach(field => registerNamedTypes(field.type, ns, registry));
+ return;
+ }
+
+ if (schema.type === 'array') {
+ registerNamedTypes(schema.items, namespace, registry);
+ } else if (schema.type === 'map') {
+ registerNamedTypes(schema.values, namespace, registry);
+ }
+};
+
+const resolve = (schema, namespace, registry) => {
+ if (typeof schema === 'string') {
+ if (PRIMITIVES.has(schema)) {
+ return { schema, namespace };
+ }
+ return (
+ registry.get(fullName(schema, namespace)) ??
+ registry.get(schema) ?? { schema, namespace, name: schema }
+ );
+ }
+ if (schema && typeof schema === 'object' && NAMED.has(schema.type) && schema.name) {
+ const ns = namespaceOf(schema, namespace);
+ return { schema, namespace: ns, name: fullName(schema.name, ns) };
+ }
+ return { schema, namespace };
+};
+
+const isRecord = schema =>
+ !!schema && typeof schema === 'object' && (schema.type === 'record' || schema.type === 'error');
+
+const typeLabel = (schema, namespace, registry) => {
+ if (Array.isArray(schema)) {
+ return schema.map(branch => typeLabel(branch, namespace, registry)).join(' | ');
+ }
+ if (typeof schema === 'string') {
+ return PRIMITIVES.has(schema) ? schema : shortName(schema);
+ }
+ if (!schema || typeof schema !== 'object') {
+ return String(schema);
+ }
+ const logical = schema.logicalType ? ` (${schema.logicalType})` : '';
+ if (NAMED.has(schema.type)) {
+ return `${shortName(schema.name)}${logical}`;
+ }
+ if (schema.type === 'array') {
+ return `array<${typeLabel(schema.items, namespace, registry)}>`;
+ }
+ if (schema.type === 'map') {
+ return `map<${typeLabel(schema.values, namespace, registry)}>`;
+ }
+ return `${typeLabel(schema.type, namespace, registry)}${logical}`;
+};
+
+/** Upper bound on listed fields: types shared at every level would otherwise grow exponentially. */
+export const MAX_FIELDS = 5000;
+
+const collect = (schema, namespace, path, registry, visiting, result) => {
+ if (result.length >= MAX_FIELDS) {
+ return;
+ }
+
+ if (Array.isArray(schema)) {
+ const records = schema.filter(branch => isRecord(resolve(branch, namespace, registry).schema));
+ schema.forEach(branch => {
+ const { schema: resolved } = resolve(branch, namespace, registry);
+ // With several record branches, the same field name can exist in more than one of them.
+ const branchPath =
+ records.length > 1 && isRecord(resolved) ? `${path}<${shortName(resolved.name)}>` : path;
+ collect(branch, namespace, branchPath, registry, visiting, result);
+ });
+ return;
+ }
+
+ const { schema: resolved, namespace: ns, name } = resolve(schema, namespace, registry);
+ if (!resolved || typeof resolved !== 'object') {
+ return;
+ }
+
+ if (resolved.type === 'array') {
+ collect(resolved.items, ns, `${path}[]`, registry, visiting, result);
+ return;
+ }
+ if (resolved.type === 'map') {
+ collect(resolved.values, ns, `${path}{}`, registry, visiting, result);
+ return;
+ }
+ if (!isRecord(resolved)) {
+ return;
+ }
+
+ // Recursive types (e.g. a linked list) would otherwise expand forever.
+ if (visiting.has(name)) {
+ return;
+ }
+ visiting.add(name);
+
+ for (const field of resolved.fields || []) {
+ if (result.length >= MAX_FIELDS) {
+ break;
+ }
+ const fieldPath = path ? `${path}.${field.name}` : field.name;
+ result.push({
+ path: fieldPath,
+ type: typeLabel(field.type, ns, registry),
+ doc: field.doc ?? ''
+ });
+ collect(field.type, ns, fieldPath, registry, visiting, result);
+ }
+
+ visiting.delete(name);
+};
+
+/**
+ * Lists every field of an Avro schema, nested ones included, as `{ path, type, doc }` in
+ * declaration order, up to MAX_FIELDS entries. Array items are addressed with `[]`, map values
+ * with `{}`, and a record branch of a union holding several records with `
`.
+ */
+export const extractAvroFields = schemaDefinition => {
+ let schema;
+ try {
+ schema = typeof schemaDefinition === 'string' ? JSON.parse(schemaDefinition) : schemaDefinition;
+ } catch (e) {
+ return [];
+ }
+
+ const registry = new Map();
+ registerNamedTypes(schema, undefined, registry);
+
+ const result = [];
+ collect(schema, undefined, '', registry, new Set(), result);
+ return result;
+};
diff --git a/client/src/utils/avroFields.test.js b/client/src/utils/avroFields.test.js
new file mode 100644
index 000000000..1a3d2951f
--- /dev/null
+++ b/client/src/utils/avroFields.test.js
@@ -0,0 +1,232 @@
+import { describe, expect, it } from 'vitest';
+import { extractAvroFields, MAX_FIELDS } from './avroFields';
+
+describe('extractAvroFields', () => {
+ it('lists every top-level field with its type and doc', () => {
+ const schema = {
+ type: 'record',
+ name: 'User',
+ fields: [
+ { name: 'id', type: 'long', doc: 'Internal user id' },
+ { name: 'email', type: ['null', 'string'], doc: 'Contact address' },
+ {
+ name: 'createdAt',
+ type: { type: 'long', logicalType: 'timestamp-millis' },
+ doc: 'Creation time'
+ },
+ { name: 'undocumented', type: 'string' }
+ ]
+ };
+
+ expect(extractAvroFields(JSON.stringify(schema))).toEqual([
+ { path: 'id', type: 'long', doc: 'Internal user id' },
+ { path: 'email', type: 'null | string', doc: 'Contact address' },
+ { path: 'createdAt', type: 'long (timestamp-millis)', doc: 'Creation time' },
+ { path: 'undocumented', type: 'string', doc: '' }
+ ]);
+ });
+
+ it('walks nested records, unions, arrays and maps', () => {
+ const schema = {
+ type: 'record',
+ name: 'Order',
+ namespace: 'com.example',
+ fields: [
+ {
+ name: 'customer',
+ type: [
+ 'null',
+ {
+ type: 'record',
+ name: 'Customer',
+ fields: [{ name: 'name', type: 'string', doc: 'Display name' }]
+ }
+ ],
+ doc: 'Who placed the order'
+ },
+ {
+ name: 'lines',
+ type: {
+ type: 'array',
+ items: {
+ type: 'record',
+ name: 'Line',
+ fields: [{ name: 'sku', type: 'string', doc: 'Product code' }]
+ }
+ }
+ },
+ {
+ name: 'attributes',
+ type: {
+ type: 'map',
+ values: {
+ type: 'record',
+ name: 'Attr',
+ fields: [{ name: 'v', type: 'int', doc: 'Value' }]
+ }
+ }
+ },
+ { name: 'status', type: { type: 'enum', name: 'Status', symbols: ['NEW', 'PAID'] } }
+ ]
+ };
+
+ expect(extractAvroFields(schema)).toEqual([
+ { path: 'customer', type: 'null | Customer', doc: 'Who placed the order' },
+ { path: 'customer.name', type: 'string', doc: 'Display name' },
+ { path: 'lines', type: 'array', doc: '' },
+ { path: 'lines[].sku', type: 'string', doc: 'Product code' },
+ { path: 'attributes', type: 'map', doc: '' },
+ { path: 'attributes{}.v', type: 'int', doc: 'Value' },
+ { path: 'status', type: 'Status', doc: '' }
+ ]);
+ });
+
+ it('resolves named type references, including across namespaces', () => {
+ const schema = {
+ type: 'record',
+ name: 'Shipment',
+ namespace: 'com.example',
+ fields: [
+ {
+ name: 'from',
+ type: {
+ type: 'record',
+ name: 'Address',
+ fields: [{ name: 'city', type: 'string', doc: 'City name' }]
+ }
+ },
+ { name: 'to', type: 'Address' },
+ { name: 'via', type: ['null', 'com.example.Address'] }
+ ]
+ };
+
+ expect(extractAvroFields(schema).map(entry => entry.path)).toEqual([
+ 'from',
+ 'from.city',
+ 'to',
+ 'to.city',
+ 'via',
+ 'via.city'
+ ]);
+ });
+
+ it('stops on recursive types', () => {
+ const schema = {
+ type: 'record',
+ name: 'Node',
+ fields: [
+ { name: 'label', type: 'string', doc: 'Label' },
+ { name: 'next', type: ['null', 'Node'], doc: 'Next node' }
+ ]
+ };
+
+ expect(extractAvroFields(schema)).toEqual([
+ { path: 'label', type: 'string', doc: 'Label' },
+ { path: 'next', type: 'null | Node', doc: 'Next node' }
+ ]);
+ });
+
+ it('resolves short names inside a referenced type against its own namespace', () => {
+ const schema = {
+ type: 'record',
+ name: 'Root',
+ namespace: 'a',
+ fields: [
+ {
+ name: 'inner',
+ type: {
+ type: 'record',
+ name: 'Inner',
+ fields: [
+ {
+ name: 'leaf',
+ type: { type: 'record', name: 'Leaf', fields: [{ name: 'v', type: 'int' }] }
+ },
+ { name: 'leaf2', type: 'Leaf' }
+ ]
+ }
+ },
+ {
+ name: 'other',
+ type: {
+ type: 'record',
+ name: 'Other',
+ namespace: 'b',
+ fields: [{ name: 'ref', type: 'a.Inner' }]
+ }
+ }
+ ]
+ };
+
+ expect(extractAvroFields(schema).map(entry => entry.path)).toContain('other.ref.leaf2.v');
+ });
+
+ it('tells apart the record branches of a union', () => {
+ const schema = {
+ type: 'record',
+ name: 'Payment',
+ fields: [
+ {
+ name: 'method',
+ type: [
+ 'null',
+ {
+ type: 'record',
+ name: 'Card',
+ fields: [{ name: 'id', type: 'string', doc: 'Card id' }]
+ },
+ { type: 'record', name: 'Bank', fields: [{ name: 'id', type: 'long', doc: 'Bank id' }] }
+ ]
+ }
+ ]
+ };
+
+ expect(extractAvroFields(schema)).toEqual([
+ { path: 'method', type: 'null | Card | Bank', doc: '' },
+ { path: 'method.id', type: 'string', doc: 'Card id' },
+ { path: 'method.id', type: 'long', doc: 'Bank id' }
+ ]);
+ });
+
+ it('shows logical types of named types', () => {
+ const schema = {
+ type: 'record',
+ name: 'Price',
+ fields: [
+ {
+ name: 'amount',
+ type: { type: 'fixed', name: 'Amount', size: 8, logicalType: 'decimal', precision: 10 }
+ }
+ ]
+ };
+
+ expect(extractAvroFields(schema)[0].type).toBe('Amount (decimal)');
+ });
+
+ it('stops listing at MAX_FIELDS when shared types blow up', () => {
+ // Every level references the next type twice, which doubles the paths at each level.
+ const levels = 20;
+ const types = [];
+ for (let i = levels; i >= 0; i--) {
+ types[i] =
+ i === levels
+ ? { type: 'record', name: `T${i}`, fields: [{ name: 'v', type: 'int' }] }
+ : {
+ type: 'record',
+ name: `T${i}`,
+ fields: [
+ { name: 'a', type: types[i + 1] },
+ { name: 'b', type: `T${i + 1}` }
+ ]
+ };
+ }
+
+ expect(extractAvroFields(types[0])).toHaveLength(MAX_FIELDS);
+ });
+
+ it('returns nothing for schemas it cannot read', () => {
+ expect(extractAvroFields('not json')).toEqual([]);
+ expect(extractAvroFields('"string"')).toEqual([]);
+ expect(extractAvroFields(undefined)).toEqual([]);
+ });
+});