From ad59bf6588d0d97422732644c6f06b80c997bf00 Mon Sep 17 00:00:00 2001 From: Emir Muhammad Date: Mon, 16 Feb 2026 19:04:49 +0100 Subject: [PATCH 1/3] Add ability to configure ers session --- src/daqpytools/apps/logging_demonstrator.py | 6 +++--- src/daqpytools/logging/logger.py | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/daqpytools/apps/logging_demonstrator.py b/src/daqpytools/apps/logging_demonstrator.py index 400f6f1..4f8f30e 100644 --- a/src/daqpytools/apps/logging_demonstrator.py +++ b/src/daqpytools/apps/logging_demonstrator.py @@ -296,7 +296,7 @@ def parse_args(self, ctx: click.Context, args: list[str]) -> None: @click.option( "-e", "--ersprotobufstream", - is_flag=True, + type=str, help=( "Set up an ERS handler, and publish to ERS" ) @@ -364,7 +364,7 @@ def main( stream_handlers: bool, child_logger: bool, disable_logger_inheritance: bool, - ersprotobufstream: bool, + ersprotobufstream: str, handlertypes:bool, handlerconf:bool, throttle: bool, @@ -384,7 +384,7 @@ def main( disable_logger_inheritance (bool): If true, disable logger inheritance so each logger instance only uses the logger handlers assigned to the given logger instance. - ersprotobufstream (bool): If true, sets up an ERS protobuf handler. Error msg + ersprotobufstream (str): Sets up an ERS protobuf handler with supplied session name. Error msg are demonstrated in the HandlerType demonstration, requiring handlerconf to be set to true. The topic for these tests is session_tester. handlertypes (bool): If true, demonstrates the advanced feature of HandlerTypes. diff --git a/src/daqpytools/logging/logger.py b/src/daqpytools/logging/logger.py index 2f99d17..bc66419 100644 --- a/src/daqpytools/logging/logger.py +++ b/src/daqpytools/logging/logger.py @@ -71,7 +71,7 @@ def get_daq_logger( rich_handler: bool = False, file_handler_path: str | None = None, stream_handlers: bool = False, - ers_kafka_handler: bool = False, + ers_kafka_handler: str | None = None, throttle: bool = False ) -> logging.Logger: """C'tor for the default logging instances. @@ -84,7 +84,7 @@ def get_daq_logger( file_handler_path (str | None): Path to the file handler log file. If None, no file handler is added. stream_handlers (bool): Whether to add both stdout and stderr stream handlers. - ers_kafka_handler (bool): Whether to add an ERS protobuf handler. + ers_kafka_handler (str): Whether to add an ERS protobuf handler. str is session name throttle (bool): Whether to add the throttle filter or not. Note, does not mean outputs are filtered by default! See ThrottleFilter for details. @@ -150,7 +150,7 @@ def get_daq_logger( add_stdout_handler(logger, use_parent_handlers) add_stderr_handler(logger, use_parent_handlers) if ers_kafka_handler: - add_ers_kafka_handler(logger, use_parent_handlers, "session_tester") + add_ers_kafka_handler(logger, use_parent_handlers, ers_kafka_handler) if throttle: # Note: Default parameters used. No functionality on customisability yet From f9311290dc8c8de9f3dde2edf7e51ec7f3adcc95 Mon Sep 17 00:00:00 2001 From: PawelPlesniak Date: Tue, 17 Feb 2026 12:12:07 +0100 Subject: [PATCH 2/3] Removing defaults that broke the nightly --- src/daqpytools/logging/handlers.py | 2 +- src/daqpytools/logging/logger.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/daqpytools/logging/handlers.py b/src/daqpytools/logging/handlers.py index 3b5a1e1..11c126a 100644 --- a/src/daqpytools/logging/handlers.py +++ b/src/daqpytools/logging/handlers.py @@ -191,7 +191,7 @@ class ERSPyLogHandlerConf: are not yet supported. """ handlers: list = field(default_factory = lambda: []) - protobufconf: ProtobufConf = field(default_factory = lambda: ProtobufConf()) + protobufconf: ProtobufConf = field(default_factory = lambda: None) @dataclass class LogHandlerConf: diff --git a/src/daqpytools/logging/logger.py b/src/daqpytools/logging/logger.py index bc66419..147f7de 100644 --- a/src/daqpytools/logging/logger.py +++ b/src/daqpytools/logging/logger.py @@ -149,7 +149,7 @@ def get_daq_logger( if stream_handlers: add_stdout_handler(logger, use_parent_handlers) add_stderr_handler(logger, use_parent_handlers) - if ers_kafka_handler: + if ers_kafka_handler: add_ers_kafka_handler(logger, use_parent_handlers, ers_kafka_handler) if throttle: From eff41b76403e380118291598e5feb3c20881e19f Mon Sep 17 00:00:00 2001 From: Emir Muhammad Date: Tue, 17 Feb 2026 12:52:29 +0100 Subject: [PATCH 3/3] Add hacky first pass at setup_ers --- src/daqpytools/apps/logging_demonstrator.py | 68 +++++++---- src/daqpytools/logging/handlers.py | 118 ++++++++++++++++++++ src/daqpytools/logging/logger.py | 31 ++++- 3 files changed, 191 insertions(+), 26 deletions(-) diff --git a/src/daqpytools/apps/logging_demonstrator.py b/src/daqpytools/apps/logging_demonstrator.py index 4f8f30e..7f9e4e1 100644 --- a/src/daqpytools/apps/logging_demonstrator.py +++ b/src/daqpytools/apps/logging_demonstrator.py @@ -401,36 +401,58 @@ def main( LoggerSetupError: If no handlers are set up for the logger. """ logger_name = "daqpytools_logging_demonstrator" + + os.environ["DUNEDAQ_ERS_WARNING"] = "erstrace,throttle,lstdout" + os.environ["DUNEDAQ_ERS_INFO"] = "lstderr,throttle,lstdout" + os.environ["DUNEDAQ_ERS_FATAL"] = "rich,lstdout" + os.environ["DUNEDAQ_ERS_ERROR"] = ( + "erstrace," + "throttle," + "lstdout," + "protobufstream(monkafka.cern.ch:30092)" + ) + main_logger: logging.Logger = get_daq_logger( logger_name=logger_name, log_level=log_level, - use_parent_handlers=not disable_logger_inheritance, - rich_handler=rich_handler, - file_handler_path=file_handler_path, - stream_handlers=stream_handlers, - ers_kafka_handler=ersprotobufstream, - throttle=throttle + rich_handler=False, + setup_ers_handlers=True, + ers_kafka_handler="session_temp" ) - if not suppress_basic: - test_main_functions(main_logger) + main_logger.warning("test") + + + # main_logger: logging.Logger = get_daq_logger( + # logger_name=logger_name, + # log_level=log_level, + # use_parent_handlers=not disable_logger_inheritance, + # rich_handler=rich_handler, + # file_handler_path=file_handler_path, + # stream_handlers=stream_handlers, + # ers_kafka_handler=ersprotobufstream, + # throttle=throttle + # ) + + # if not suppress_basic: + # test_main_functions(main_logger) - if child_logger: - test_child_logger( - logger_name, - log_level, - disable_logger_inheritance, - rich_handler, - file_handler_path, - stream_handlers - ) + # if child_logger: + # test_child_logger( + # logger_name, + # log_level, + # disable_logger_inheritance, + # rich_handler, + # file_handler_path, + # stream_handlers + # ) - if throttle: - test_throttle(main_logger) - if handlertypes: - test_handlertypes(main_logger) - if handlerconf: - test_handlerconf(main_logger) + # if throttle: + # test_throttle(main_logger) + # if handlertypes: + # test_handlertypes(main_logger) + # if handlerconf: + # test_handlerconf(main_logger) if __name__ == "__main__": diff --git a/src/daqpytools/logging/handlers.py b/src/daqpytools/logging/handlers.py index 11c126a..c8c40e6 100644 --- a/src/daqpytools/logging/handlers.py +++ b/src/daqpytools/logging/handlers.py @@ -275,6 +275,7 @@ def _convert_str_to_handlertype(handler_str: str) -> tuple[HandlerType, converts "protobufstream(url:port)" to return both the HandlerType and the protobuf configuration """ + # print(f"{handler_str=}") if "erstrace" in handler_str: msg = ( "ERSTrace is a C++ implementation, " @@ -297,10 +298,12 @@ def _make_ers_handler_conf(ers_log_level :str) -> ERSPyLogHandlerConf: """Generates the ERSPyLogHandlerConf from reading an environment variable.""" erspyloghandlerconf = ERSPyLogHandlerConf() envvalue = os.getenv(ers_log_level) + # print(f"{envvalue=}") if envvalue is None: raise ERSEnvError(ers_log_level) for h in envvalue.split(","): + # print(f"{h=}") handlertype, kafkaconf = LogHandlerConf._convert_str_to_handlertype(h) erspyloghandlerconf.handlers.append(handlertype) if kafkaconf: @@ -543,6 +546,19 @@ def _format_timestamp(timestamp: float) -> str: padding: int = LOG_RECORD_PADDING.get("time", 25) time_str: str = dt.strftime(DATE_TIME_BASE_FORMAT).ljust(padding)[:padding] return Text(time_str, style="logging.time") + + +def add_throttle_filter(log: logging.Logger) -> None: + """Add the Throttle filter to the logger. + + Args: + log (logging.Logger): Logger to add the rich handler to. + + Returns: + None + """ + log.addFilter(ThrottleFilter()) + return def check_parent_handlers( log: logging.Logger, @@ -717,3 +733,105 @@ def add_file_handler(log: logging.Logger, use_parent_handlers: bool, path: str) log.addHandler(file_handler) return + +def _logger_has_handler( + log: logging.Logger, + handler_type: type[logging.Handler], + target_stream: io.IOBase | None = None, +) -> bool: + """Check if logger already has a matching handler. + + For StreamHandler, ``target_stream`` can be used to distinguish stdout/stderr. + """ + type_matches = [isinstance(handler, handler_type) for handler in log.handlers] + stream_matches = [ + handler.stream is target_stream if target_stream else False + for handler in log.handlers + if isinstance(handler, logging.StreamHandler) + ] + return any(type_matches + stream_matches) + + +def _logger_has_filter(log: logging.Logger, filter_type: type[logging.Filter]) -> bool: + """Check if logger already has a matching filter type.""" + return any(isinstance(logger_filter, filter_type) for logger_filter in log.filters) + + +def add_handlers_from_types( + log: logging.Logger, + handler_types: set[HandlerType], + ers_session_name: str | None, +) -> None: + """Add handlers to a logger based on HandlerType values. + + This helper intentionally supports only the default options for now: + - ``use_parent_handlers`` is always True. + - ``HandlerType.File`` is not supported and raises immediately. + - ``HandlerType.Protobufstream`` requires ``ers_session_name``. + """ + if HandlerType.File in handler_types: + err_msg = "HandlerType.File is not supported by add_handlers_from_types" + raise ValueError(err_msg) + + if HandlerType.Protobufstream in handler_types and not ers_session_name: + err_msg = "ers_session_name is required for HandlerType.Protobufstream" + raise ValueError(err_msg) + + effective_handler_types = set(handler_types) + if HandlerType.Stream in effective_handler_types: + effective_handler_types.update({HandlerType.Lstdout, HandlerType.Lstderr}) + + existing_stream_handlers = { + HandlerType.Lstdout + if _logger_has_handler( + log, logging.StreamHandler, target_stream=cast(io.IOBase, sys.stdout) + ) + else None, + HandlerType.Lstderr + if _logger_has_handler( + log, logging.StreamHandler, target_stream=cast(io.IOBase, sys.stderr) + ) + else None, + } + existing_stream_handlers.discard(None) + + existing_handlers = { + HandlerType.Rich if _logger_has_handler(log, FormattedRichHandler) else None, + HandlerType.Protobufstream + if _logger_has_handler(log, ERSKafkaLogHandler) + else None, + HandlerType.Throttle if _logger_has_filter(log, ThrottleFilter) else None, + } + existing_handlers.discard(None) + existing_handlers.update(existing_stream_handlers) + + dispatch = { + HandlerType.Rich: lambda: add_rich_handler(log, True), + HandlerType.Lstdout: lambda: add_stdout_handler(log, True), + HandlerType.Lstderr: lambda: add_stderr_handler(log, True), + HandlerType.Protobufstream: lambda: add_ers_kafka_handler( + log, True, ers_session_name + ), + HandlerType.Throttle: lambda: add_throttle_filter(log) + } + + #! Try to revisit this logic + + install_order = [ + HandlerType.Rich, + HandlerType.Lstdout, + HandlerType.Lstderr, + HandlerType.Protobufstream, + HandlerType.Throttle, + ] + + for handler_type in install_order: + if handler_type not in effective_handler_types: + continue + if handler_type in existing_handlers: + continue + installer = dispatch.get(handler_type) + if installer is None: + continue + installer() + diff --git a/src/daqpytools/logging/logger.py b/src/daqpytools/logging/logger.py index 147f7de..3f0c1fa 100644 --- a/src/daqpytools/logging/logger.py +++ b/src/daqpytools/logging/logger.py @@ -8,7 +8,9 @@ from daqpytools.logging.exceptions import LoggerSetupError from daqpytools.logging.handlers import ( - ThrottleFilter, + LogHandlerConf, + add_handlers_from_types, + add_throttle_filter, add_ers_kafka_handler, add_file_handler, add_rich_handler, @@ -72,7 +74,9 @@ def get_daq_logger( file_handler_path: str | None = None, stream_handlers: bool = False, ers_kafka_handler: str | None = None, - throttle: bool = False + throttle: bool = False, + + setup_ers_handlers: bool = False, ) -> logging.Logger: """C'tor for the default logging instances. @@ -154,7 +158,28 @@ def get_daq_logger( if throttle: # Note: Default parameters used. No functionality on customisability yet - logger.addFilter(ThrottleFilter()) + add_throttle_filter(logger) + + + + if setup_ers_handlers: + + # need to grab the list of relevant handlers that exist in ERS + #! This is very dependent on ERS env variables existing!!! + lhc_conf = LogHandlerConf._get_oks_conf() + all_handlers = {handler for handler_conf in lhc_conf.values() for handler in handler_conf.handlers} + + print(all_handlers) + + add_handlers_from_types(logger, all_handlers, ers_kafka_handler) + + # now what.. Well we have a list of handlers to add now huh.. + + + + + + # Set log level for all handlers if requested if log_level is not logging.NOTSET: