Move QMP-reachable type tracking and name masking out of the introspect visitor into a standalone QAPISchemaUsedTypes visitor in a new schema_analysis module. Run the analysis pass in QAPICBackend.generate() in preparation for other generators.
While at it, refactor a bit the code to make it easier to read, and optimize using _used_type_set for O(1) lookups. Signed-off-by: Marc-André Lureau <[email protected]> --- meson.build | 1 + scripts/qapi/backend.py | 7 +- scripts/qapi/introspect.py | 78 +++++-------------- scripts/qapi/schema_analysis.py | 164 ++++++++++++++++++++++++++++++++++++++++ 4 files changed, 191 insertions(+), 59 deletions(-) diff --git a/meson.build b/meson.build index 49a5baf5b52f..6ddb323c13f5 100644 --- a/meson.build +++ b/meson.build @@ -3499,6 +3499,7 @@ qapi_gen_depends = [ meson.current_source_dir() / 'scripts/qapi/__init__.py', meson.current_source_dir() / 'scripts/qapi/main.py', meson.current_source_dir() / 'scripts/qapi/parser.py', meson.current_source_dir() / 'scripts/qapi/schema.py', + meson.current_source_dir() / 'scripts/qapi/schema_analysis.py', meson.current_source_dir() / 'scripts/qapi/source.py', meson.current_source_dir() / 'scripts/qapi/types.py', meson.current_source_dir() / 'scripts/qapi/visit.py', diff --git a/scripts/qapi/backend.py b/scripts/qapi/backend.py index 49ae6ecdd33e..59329965890f 100644 --- a/scripts/qapi/backend.py +++ b/scripts/qapi/backend.py @@ -8,6 +8,7 @@ from .features import gen_features from .introspect import gen_introspect from .schema import QAPISchema +from .schema_analysis import QAPISchemaUsedTypes from .types import gen_types from .visit import gen_visit @@ -49,7 +50,7 @@ def generate(self, """ Generate C code for the given schema into the target directory. - :param schema_file: The primary QAPI schema file. + :param schema: The primary QAPI schema file. :param output_dir: The output directory to store generated code. :param prefix: Optional C-code prefix for symbol names. :param unmask: Expose non-ABI names through introspection? @@ -57,9 +58,11 @@ def generate(self, :raise QAPIError: On failures. """ + schema_types = QAPISchemaUsedTypes(unmask) + schema.visit(schema_types) gen_types(schema, output_dir, prefix, builtins) gen_features(schema, output_dir, prefix) gen_visit(schema, output_dir, prefix, builtins) gen_commands(schema, output_dir, prefix, gen_tracing) gen_events(schema, output_dir, prefix) - gen_introspect(schema, output_dir, prefix, unmask) + gen_introspect(schema, output_dir, prefix, schema_types) diff --git a/scripts/qapi/introspect.py b/scripts/qapi/introspect.py index 7e28de2279ad..9e76e3aa38a9 100644 --- a/scripts/qapi/introspect.py +++ b/scripts/qapi/introspect.py @@ -28,9 +28,7 @@ from .schema import ( QAPISchema, QAPISchemaAlternatives, - QAPISchemaArrayType, QAPISchemaBranches, - QAPISchemaBuiltinType, QAPISchemaEntity, QAPISchemaEnumMember, QAPISchemaFeature, @@ -40,6 +38,7 @@ QAPISchemaType, QAPISchemaVariant, ) +from .schema_analysis import QAPISchemaUsedTypes from .source import QAPISourceInfo @@ -169,15 +168,13 @@ def to_c_string(string: str) -> str: class QAPISchemaGenIntrospectVisitor(QAPISchemaMonolithicCVisitor): - def __init__(self, prefix: str, unmask: bool): + def __init__(self, prefix: str, schema_types: QAPISchemaUsedTypes): super().__init__( prefix, 'qapi-introspect', ' * QAPI/QMP schema introspection', __doc__) - self._unmask = unmask + self._schema_types = schema_types self._schema: Optional[QAPISchema] = None self._trees: List[Annotated[SchemaInfo]] = [] - self._used_types: List[QAPISchemaType] = [] - self._name_map: Dict[str, str] = {} self._genc.add(mcgen(''' #include "qemu/osdep.h" #include "%(prefix)sqapi-introspect.h" @@ -190,7 +187,7 @@ def visit_begin(self, schema: QAPISchema) -> None: def visit_end(self) -> None: # visit the types that are actually used - for typ in self._used_types: + for typ in self._schema_types.used_types(): typ.visit(self) # generate C name = c_name(self._prefix, protect=False) + 'qmp_schema_qlit' @@ -207,45 +204,11 @@ def visit_end(self) -> None: c_string=_tree_to_qlit(self._trees))) self._schema = None self._trees = [] - self._used_types = [] - self._name_map = {} def visit_needed(self, entity: QAPISchemaEntity) -> bool: # Ignore types on first pass; visit_end() will pick up used types return not isinstance(entity, QAPISchemaType) - def _name(self, name: str) -> str: - if self._unmask: - return name - if name not in self._name_map: - self._name_map[name] = '%d' % len(self._name_map) - return self._name_map[name] - - def _use_type(self, typ: QAPISchemaType) -> str: - assert self._schema is not None - - # Map the various integer types to plain int - if typ.json_type() == 'int': - type_int = self._schema.lookup_type('int') - assert type_int - typ = type_int - elif (isinstance(typ, QAPISchemaArrayType) and - typ.element_type.json_type() == 'int'): - type_intlist = self._schema.lookup_type('intList') - assert type_intlist - typ = type_intlist - # Add type to work queue if new - if typ not in self._used_types: - self._used_types.append(typ) - # Clients should examine commands and events, not types. Hide - # type names as integers to reduce the temptation. Also, it - # saves a few characters on the wire. - if isinstance(typ, QAPISchemaBuiltinType): - return typ.name - if isinstance(typ, QAPISchemaArrayType): - return '[' + self._use_type(typ.element_type) + ']' - return self._name(typ.name) - @staticmethod def _gen_features(features: Sequence[QAPISchemaFeature] ) -> List[Annotated[str]]: @@ -267,11 +230,10 @@ def _gen_tree(self, name: str, mtype: str, obj: Dict[str, object], """ comment: Optional[str] = None if mtype not in ('command', 'event', 'builtin', 'array'): - if not self._unmask: - # Output a comment to make it easy to map masked names - # back to the source when reading the generated output. - comment = f'"{self._name(name)}" = {name}' - name = self._name(name) + masked = self._schema_types.masked_name(name) + if masked != name: + comment = f'"{masked}" = {name}' + name = masked obj['name'] = name obj['meta-type'] = mtype if features: @@ -291,7 +253,7 @@ def _gen_object_member(self, member: QAPISchemaObjectTypeMember ) -> Annotated[SchemaInfoObjectMember]: obj: SchemaInfoObjectMember = { 'name': member.name, - 'type': self._use_type(member.type) + 'type': self._schema_types.introspection_name(member.type) } if member.optional: obj['default'] = None @@ -303,7 +265,7 @@ def _gen_variant(self, variant: QAPISchemaVariant ) -> Annotated[SchemaInfoObjectVariant]: obj: SchemaInfoObjectVariant = { 'case': variant.name, - 'type': self._use_type(variant.type) + 'type': self._schema_types.introspection_name(variant.type) } return Annotated(obj, variant.ifcond) @@ -326,7 +288,7 @@ def visit_enum_type(self, name: str, info: Optional[QAPISourceInfo], def visit_array_type(self, name: str, info: Optional[QAPISourceInfo], ifcond: QAPISchemaIfCond, element_type: QAPISchemaType) -> None: - element = self._use_type(element_type) + element = self._schema_types.introspection_name(element_type) self._gen_tree('[' + element + ']', 'array', {'element-type': element}, ifcond) @@ -349,8 +311,9 @@ def visit_alternate_type(self, name: str, info: Optional[QAPISourceInfo], alternatives: QAPISchemaAlternatives) -> None: self._gen_tree( name, 'alternate', - {'members': [Annotated({'type': self._use_type(m.type)}, - m.ifcond) + {'members': [Annotated({ + 'type': self._schema_types.introspection_name(m.type) + }, m.ifcond) for m in alternatives.variants]}, ifcond, features ) @@ -367,8 +330,8 @@ def visit_command(self, name: str, info: Optional[QAPISourceInfo], arg_type = arg_type or self._schema.the_empty_object_type ret_type = ret_type or self._schema.the_empty_object_type obj: SchemaInfoCommand = { - 'arg-type': self._use_type(arg_type), - 'ret-type': self._use_type(ret_type) + 'arg-type': self._schema_types.introspection_name(arg_type), + 'ret-type': self._schema_types.introspection_name(ret_type) } if allow_oob: obj['allow-oob'] = allow_oob @@ -382,12 +345,13 @@ def visit_event(self, name: str, info: Optional[QAPISourceInfo], assert self._schema is not None arg_type = arg_type or self._schema.the_empty_object_type - self._gen_tree(name, 'event', {'arg-type': self._use_type(arg_type)}, - ifcond, features) + self._gen_tree(name, 'event', { + 'arg-type': self._schema_types.introspection_name(arg_type) + }, ifcond, features) def gen_introspect(schema: QAPISchema, output_dir: str, prefix: str, - opt_unmask: bool) -> None: - vis = QAPISchemaGenIntrospectVisitor(prefix, opt_unmask) + schema_types: QAPISchemaUsedTypes) -> None: + vis = QAPISchemaGenIntrospectVisitor(prefix, schema_types) schema.visit(vis) vis.write(output_dir) diff --git a/scripts/qapi/schema_analysis.py b/scripts/qapi/schema_analysis.py new file mode 100644 index 000000000000..7e42abbc14e1 --- /dev/null +++ b/scripts/qapi/schema_analysis.py @@ -0,0 +1,164 @@ +# SPDX-License-Identifier: GPL-2.0-or-later +""" +Collect introspectable types from a QAPI schema and assign masked names. + +Copyright (C) 2015-2026 Red Hat, Inc. + +Authors: + Markus Armbruster <[email protected]> + John Snow <[email protected]> + Marc-André Lureau <[email protected]> +""" + +from typing import ( + Dict, + List, + Optional, + Sequence, + Set, +) + +from .schema import ( + QAPISchema, + QAPISchemaAlternatives, + QAPISchemaArrayType, + QAPISchemaBranches, + QAPISchemaBuiltinType, + QAPISchemaEntity, + QAPISchemaFeature, + QAPISchemaIfCond, + QAPISchemaObjectType, + QAPISchemaObjectTypeMember, + QAPISchemaType, + QAPISchemaVisitor, +) +from .source import QAPISourceInfo + + +class QAPISchemaUsedTypes(QAPISchemaVisitor): + """Collect the set of QMP-reachable types from a schema. + + Types are discovered transitively starting from commands and events. + Each type is also given a masked introspection name (an integer + string). + """ + + def __init__(self, unmask: bool): + self._unmask = unmask + self._schema: Optional[QAPISchema] = None + # Ordered list + set: insert during iteration + O(1) check + self._used_types: List[QAPISchemaType] = [] + self._used_types_set: Set[QAPISchemaType] = set() + self._name_map: Dict[str, str] = {} + + def visit_begin(self, schema: QAPISchema) -> None: + self._schema = schema + self._used_types = [] + self._used_types_set = set() + self._name_map = {} + + def visit_end(self) -> None: + assert self._schema is not None + # Discover transitively-used types; the list grows as + # visiting each type registers the types it references. + for typ in self._used_types: + typ.visit(self) + # Assign stable masked names now that all types are known + counter = 0 + for typ in self._used_types: + if isinstance(typ, (QAPISchemaBuiltinType, QAPISchemaArrayType)): + continue + self._name_map[typ.name] = ( + typ.name if self._unmask else str(counter)) + counter += 1 + + def visit_needed(self, entity: QAPISchemaEntity) -> bool: + # Skip types during main traversal; visit_end() handles them + return not isinstance(entity, QAPISchemaType) + + def visit_command(self, name: str, info: Optional[QAPISourceInfo], + ifcond: QAPISchemaIfCond, + features: List[QAPISchemaFeature], + arg_type: Optional[QAPISchemaObjectType], + ret_type: Optional[QAPISchemaType], gen: bool, + success_response: bool, boxed: bool, allow_oob: bool, + allow_preconfig: bool, coroutine: bool) -> None: + assert self._schema is not None + self._register_type(arg_type or self._schema.the_empty_object_type) + self._register_type(ret_type or self._schema.the_empty_object_type) + + def visit_event(self, name: str, info: Optional[QAPISourceInfo], + ifcond: QAPISchemaIfCond, + features: List[QAPISchemaFeature], + arg_type: Optional[QAPISchemaObjectType], + boxed: bool) -> None: + assert self._schema is not None + self._register_type(arg_type or self._schema.the_empty_object_type) + + def visit_object_type_flat( + self, name: str, info: Optional[QAPISourceInfo], + ifcond: QAPISchemaIfCond, + features: List[QAPISchemaFeature], + members: List[QAPISchemaObjectTypeMember], + branches: Optional[QAPISchemaBranches]) -> None: + for m in members: + self._register_type(m.type) + if branches: + for v in branches.variants: + self._register_type(v.type) + + def visit_array_type(self, name: str, info: Optional[QAPISourceInfo], + ifcond: QAPISchemaIfCond, + element_type: QAPISchemaType) -> None: + self._register_type(element_type) + + def visit_alternate_type( + self, name: str, info: Optional[QAPISourceInfo], + ifcond: QAPISchemaIfCond, + features: List[QAPISchemaFeature], + alternatives: QAPISchemaAlternatives) -> None: + for m in alternatives.variants: + self._register_type(m.type) + + def _register_type(self, typ: QAPISchemaType) -> None: + """Record a type as QMP-reachable (idempotent).""" + typ = self._canonicalize_type(typ) + if typ not in self._used_types_set: + self._used_types.append(typ) + self._used_types_set.add(typ) + if isinstance(typ, QAPISchemaArrayType): + self._register_type(typ.element_type) + + def _canonicalize_type(self, typ: QAPISchemaType) -> QAPISchemaType: + """Canonicalize integer types to plain int.""" + assert self._schema is not None + if typ.json_type() == 'int': + type_int = self._schema.lookup_type('int') + assert type_int + return type_int + if (isinstance(typ, QAPISchemaArrayType) and + typ.element_type.json_type() == 'int'): + type_intlist = self._schema.lookup_type('intList') + assert type_intlist + return type_intlist + return typ + + def masked_name(self, name: str) -> str: + """Return the masked name for a non-builtin, non-array type.""" + assert name in self._name_map, \ + f"type '{name}' was not registered or is builtin/array" + return self._name_map[name] + + def introspection_name(self, typ: QAPISchemaType) -> str: + """Return the introspection name for a type.""" + typ = self._canonicalize_type(typ) + if isinstance(typ, QAPISchemaBuiltinType): + return typ.name + if isinstance(typ, QAPISchemaArrayType): + return '[' + self.introspection_name(typ.element_type) + ']' + assert typ in self._used_types_set + return self.masked_name(typ.name) + + def used_types(self) -> Sequence[QAPISchemaType]: + """Return the types to include in QAPI introspection.""" + return self._used_types -- 2.55.0.543.g5ebe2ebe4ea8
