diff --git a/src/daqpytools/apps/logging_demonstrator.py b/src/daqpytools/apps/logging_demonstrator.py index 400f6f1..b87c615 100644 --- a/src/daqpytools/apps/logging_demonstrator.py +++ b/src/daqpytools/apps/logging_demonstrator.py @@ -10,9 +10,11 @@ from daqpytools.logging.handlers import ( HandlerType, LogHandlerConf, + add_stdout_handler, ) + from daqpytools.logging.levels import logging_log_level_keys -from daqpytools.logging.logger import get_daq_logger +from daqpytools.logging.logger import get_daq_logger, setup_daq_ers_logger from daqpytools.logging.utils import get_width @@ -296,7 +298,7 @@ def parse_args(self, ctx: click.Context, args: list[str]) -> None: @click.option( "-e", "--ersprotobufstream", - is_flag=True, + type=str, help=( "Set up an ERS handler, and publish to ERS" ) @@ -364,7 +366,7 @@ def main( stream_handlers: bool, child_logger: bool, disable_logger_inheritance: bool, - ersprotobufstream: bool, + ersprotobufstream: str, handlertypes:bool, handlerconf:bool, throttle: bool, @@ -384,7 +386,7 @@ def main( disable_logger_inheritance (bool): If true, disable logger inheritance so each logger instance only uses the logger handlers assigned to the given logger instance. - ersprotobufstream (bool): If true, sets up an ERS protobuf handler. Error msg + ersprotobufstream (str): Sets up an ERS protobuf handler with supplied session name. Error msg are demonstrated in the HandlerType demonstration, requiring handlerconf to be set to true. The topic for these tests is session_tester. handlertypes (bool): If true, demonstrates the advanced feature of HandlerTypes. @@ -401,36 +403,94 @@ def main( LoggerSetupError: If no handlers are set up for the logger. """ logger_name = "daqpytools_logging_demonstrator" + + os.environ["DUNEDAQ_ERS_WARNING"] = "erstrace,throttle,lstderr" + os.environ["DUNEDAQ_ERS_INFO"] = "lstderr,throttle,lstderr" + os.environ["DUNEDAQ_ERS_FATAL"] = "lstderr" + os.environ["DUNEDAQ_ERS_ERROR"] = ( + "erstrace," + "throttle," + "lstderr," + "protobufstream(monkafka.cern.ch:30092)" + ) + + handlerconf = LogHandlerConf(init_ers=True) + main_logger: logging.Logger = get_daq_logger( logger_name=logger_name, log_level=log_level, - use_parent_handlers=not disable_logger_inheritance, - rich_handler=rich_handler, - file_handler_path=file_handler_path, - stream_handlers=stream_handlers, - ers_kafka_handler=ersprotobufstream, - throttle=throttle + stream_handlers=False, + rich_handler=True # only rich was defined ) - if not suppress_basic: - test_main_functions(main_logger) + main_logger.warning("Only Rich") + + # add_stdout_handler(main_logger, True) + setup_daq_ers_logger(main_logger, "session_temp") + + main_logger.critical("Should be only rich") + + + # main_logger.critical("test") #use only rich because we only iniitlaise with rich + + main_logger.critical("Stream", extra={"handlers": [HandlerType.Stream]}) + - if child_logger: - test_child_logger( - logger_name, - log_level, - disable_logger_inheritance, - rich_handler, - file_handler_path, - stream_handlers - ) + + main_logger.critical("ERS (lstderr only)", extra=handlerconf.ERS) + + + + + + # define a default handlerconf so for example + + + """ + Concrete suggestions + + For now: + get_daq_logger = rich_handler = True # save rich_handler and set as base class + + setup_ers(log) #adds stream handler and what have you + + log.warning("something") # only goes to rich because we only initialise with rich + + log.warning("something else", extra= ers) # use whatever is in ers + + """ + + + # main_logger: logging.Logger = get_daq_logger( + # logger_name=logger_name, + # log_level=log_level, + # use_parent_handlers=not disable_logger_inheritance, + # rich_handler=rich_handler, + # file_handler_path=file_handler_path, + # stream_handlers=stream_handlers, + # ers_kafka_handler=ersprotobufstream, + # throttle=throttle + # ) + + # if not suppress_basic: + # test_main_functions(main_logger) + + # if child_logger: + # test_child_logger( + # logger_name, + # log_level, + # disable_logger_inheritance, + # rich_handler, + # file_handler_path, + # stream_handlers + # ) - if throttle: - test_throttle(main_logger) - if handlertypes: - test_handlertypes(main_logger) - if handlerconf: - test_handlerconf(main_logger) + # if throttle: + # test_throttle(main_logger) + # if handlertypes: + # test_handlertypes(main_logger) + # if handlerconf: + # test_handlerconf(main_logger) if __name__ == "__main__": diff --git a/src/daqpytools/logging/handlers.py b/src/daqpytools/logging/handlers.py index 3b5a1e1..4b80972 100644 --- a/src/daqpytools/logging/handlers.py +++ b/src/daqpytools/logging/handlers.py @@ -191,8 +191,9 @@ class ERSPyLogHandlerConf: are not yet supported. """ handlers: list = field(default_factory = lambda: []) - protobufconf: ProtobufConf = field(default_factory = lambda: ProtobufConf()) + protobufconf: ProtobufConf = field(default_factory = lambda: None) +#! TODO/now= consider @dataclass(frozen=True) @dataclass class LogHandlerConf: """Dataclass that holds the various streams and relevant handlers. @@ -341,8 +342,9 @@ class BaseHandlerFilter(logging.Filter): """Base filter that hold the logic on choosing if a handler should emit based on what HandlersTypes are supplied to it. """ - def __init__(self) -> None: + def __init__(self, default_case = LogHandlerConf.get_base() ) -> None: """C'tor.""" + self.default_case = default_case super().__init__() def get_allowed(self, record: logging.LogRecord) -> list | None: @@ -367,7 +369,7 @@ def get_allowed(self, record: logging.LogRecord) -> list | None: # Handle the non-ERS case else: - allowed = getattr(record, "handlers", LogHandlerConf.get_base()) + allowed = getattr(record, "handlers", self.default_case) return allowed class HandleIDFilter(BaseHandlerFilter): @@ -375,11 +377,13 @@ class HandleIDFilter(BaseHandlerFilter): if the current handler (defined by the handler_id) is within the set of allowed handlers. """ - def __init__(self, handler_id: HandlerType | list[HandlerType]) -> None: + def __init__(self, handler_id: HandlerType | list[HandlerType], default_case = LogHandlerConf.get_base()) -> None: """Initialises HandleIDFilter with the handler_id, to identify what kind of handler this filter is. """ - super().__init__() + super().__init__( + default_case = default_case + ) # Normalise handler_id to be a set if isinstance(handler_id, list): @@ -416,9 +420,11 @@ class ThrottleFilter(BaseHandlerFilter): ... logger.error("Repeated error message") """ - def __init__(self, initial_threshold: int = 30, time_limit: int = 30) -> None: + def __init__(self, default_case=LogHandlerConf.get_base(), initial_threshold: int = 30, time_limit: int = 30) -> None: """C'tor.""" - super().__init__() + super().__init__( + default_case = default_case + ) self.initial_threshold = initial_threshold self.time_limit = time_limit self.issue_map: dict[str, IssueRecord] = defaultdict(IssueRecord) @@ -543,6 +549,38 @@ def _format_timestamp(timestamp: float) -> str: padding: int = LOG_RECORD_PADDING.get("time", 25) time_str: str = dt.strftime(DATE_TIME_BASE_FORMAT).ljust(padding)[:padding] return Text(time_str, style="logging.time") + + +def add_throttle_filter(log: logging.Logger, default_case = {HandlerType.Throttle}) -> None: + """Add the Throttle filter to the logger. + + Args: + log (logging.Logger): Logger to add the rich handler to. + + Returns: + None + """ + log.addFilter(ThrottleFilter(default_case=default_case)) + return + +def _logger_has_handler( + log: logging.Logger, + handler_type: type[logging.Handler], + target_stream: io.IOBase | None = None, +) -> bool: + """Check if logger already has a matching handler. + + For StreamHandler, ``target_stream`` can be used to distinguish stdout/stderr. + """ + + type_matches = [isinstance(handler, handler_type) for handler in log.handlers if not isinstance(handler, logging.StreamHandler)] + + stream_matches = [ + handler.stream is target_stream if target_stream else False + for handler in log.handlers + if isinstance(handler, logging.StreamHandler) + ] + return any(type_matches + stream_matches) def check_parent_handlers( log: logging.Logger, @@ -572,8 +610,8 @@ def check_parent_handlers( # Check that we are not using the true logging root logger python_root_logger_name = logging.getLogger().name if log.name == python_root_logger_name: - err_nsg = "You should not be interfacing with the root logger" - raise ValueError(err_nsg) + err_msg = "You should not be interfacing with the root logger" + raise ValueError(err_msg) # Validate the stream handler has a target stream if handler_type.__name__ == "StreamHandler" and target_stream is None: err_msg = ( @@ -591,22 +629,14 @@ def check_parent_handlers( logger_parent = log.parent this_is_root_logger = logger_parent.name == python_root_logger_name while not this_is_root_logger: - handler_checking = [ - isinstance(handler, handler_type) for handler in logger_parent.handlers - ] - stream_handler_checking = [ - handler.stream is target_stream if target_stream else False - for handler in logger_parent.handlers - if isinstance(handler, logging.StreamHandler) - ] - if any(handler_checking + stream_handler_checking): + if _logger_has_handler(logger_parent,handler_type, target_stream): raise LoggerHandlerError(logger_parent.name, handler_type) logger_parent = logger_parent.parent this_is_root_logger = logger_parent.name == python_root_logger_name return -def add_rich_handler(log: logging.Logger, use_parent_handlers: bool) -> None: +def add_rich_handler(log: logging.Logger, use_parent_handlers: bool, default_case={HandlerType.Rich}) -> None: """Add a rich handler to the logger. Args: @@ -621,13 +651,19 @@ def add_rich_handler(log: logging.Logger, use_parent_handlers: bool) -> None: """ check_parent_handlers(log, use_parent_handlers, FormattedRichHandler) width: int = get_width() - handler: RichHandler = FormattedRichHandler(width=width) - handler.addFilter(HandleIDFilter(HandlerType.Rich)) + handler: RichHandler = FormattedRichHandler(width=width) + + handler.addFilter( + HandleIDFilter( + handler_id=HandlerType.Rich, + default_case=default_case + ) + ) log.addHandler(handler) return - + def add_ers_kafka_handler(log: logging.Logger, use_parent_handlers: bool, - session_name:str, topic: str = "ers_stream", + session_name:str, default_case = {HandlerType.Protobufstream}, topic: str = "ers_stream", address: str ="monkafka.cern.ch:30092") -> None: # TODO/future: topic and address are new, propagate to all relevant implementation """Add an ers protobuf handler to the root logger.""" @@ -636,10 +672,16 @@ def add_ers_kafka_handler(log: logging.Logger, use_parent_handlers: bool, kafka_address = address, kafka_topic = topic ) - handler.addFilter(HandleIDFilter(HandlerType.Protobufstream)) + + handler.addFilter( + HandleIDFilter( + handler_id=HandlerType.Protobufstream, + default_case=default_case + ) + ) log.addHandler(handler) -def add_stdout_handler(log: logging.Logger, use_parent_handlers: bool) -> None: +def add_stdout_handler(log: logging.Logger, use_parent_handlers: bool, default_case={HandlerType.Stream, HandlerType.Lstdout}) -> None: """Add a stdout handler to the logger. Args: @@ -660,12 +702,18 @@ def add_stdout_handler(log: logging.Logger, use_parent_handlers: bool) -> None: ) stdout_handler = logging.StreamHandler(sys.stdout) stdout_handler.setFormatter(LoggingFormatter()) - stdout_handler.addFilter(HandleIDFilter([HandlerType.Stream, HandlerType.Lstdout])) + + stdout_handler.addFilter( + HandleIDFilter( + handler_id=[HandlerType.Stream, HandlerType.Lstdout], + default_case=default_case + ) + ) log.addHandler(stdout_handler) return - -def add_stderr_handler(log: logging.Logger, use_parent_handlers: bool) -> None: +# Consider seeing if there is a way to generalify the add X handler.. +def add_stderr_handler(log: logging.Logger, use_parent_handlers: bool, default_case={HandlerType.Lstderr, HandlerType.Stream}) -> None: """Add a stderr handler to the logger. The error is set to the ERROR level, and will only log messages at that level @@ -690,13 +738,18 @@ def add_stderr_handler(log: logging.Logger, use_parent_handlers: bool) -> None: ) stderr_handler = logging.StreamHandler(sys.stderr) stderr_handler.setFormatter(LoggingFormatter()) - stderr_handler.addFilter(HandleIDFilter([HandlerType.Stream, HandlerType.Lstderr])) + stderr_handler.addFilter( + HandleIDFilter( + handler_id=[HandlerType.Stream, HandlerType.Lstderr], + default_case=default_case + ) + ) stderr_handler.setLevel(logging.ERROR) log.addHandler(stderr_handler) return -def add_file_handler(log: logging.Logger, use_parent_handlers: bool, path: str) -> None: +def add_file_handler(log: logging.Logger, use_parent_handlers: bool, path: str, default_case={HandlerType.File}) -> None: """Add a file handler to the root logger. Args: @@ -713,7 +766,105 @@ def add_file_handler(log: logging.Logger, use_parent_handlers: bool, path: str) check_parent_handlers(log, use_parent_handlers, logging.FileHandler) file_handler = logging.FileHandler(filename=path) file_handler.setFormatter(LoggingFormatter()) - file_handler.addFilter(HandleIDFilter(HandlerType.File)) + file_handler.addFilter( + HandleIDFilter( + handler_id=HandlerType.File, + default_case=default_case + ) + ) log.addHandler(file_handler) return + +def _logger_has_filter(log: logging.Logger, filter_type: type[logging.Filter]) -> bool: + """Check if logger already has a matching filter type.""" + return any(isinstance(logger_filter, filter_type) for logger_filter in log.filters) + + +def add_handlers_from_types( + log: logging.Logger, + handler_types: set[HandlerType], + ers_session_name: str | None, +) -> None: + """Add handlers to a logger based on a set of HandlerType values. + + This helper intentionally supports only the default options for now: + - ``use_parent_handlers`` is always True. + - ``HandlerType.File`` is not supported and raises immediately. + - ``HandlerType.Protobufstream`` requires ``ers_session_name``. + """ + + default_case = {HandlerType.Unknown} + if HandlerType.File in handler_types: + err_msg = "HandlerType.File is not supported by add_handlers_from_types" + raise ValueError(err_msg) + + if HandlerType.Protobufstream in handler_types and not ers_session_name: + err_msg = "ers_session_name is required for HandlerType.Protobufstream" + raise ValueError(err_msg) + + # Update relevant handler types that was parsed + effective_handler_types = set(handler_types) + if HandlerType.Stream in effective_handler_types: + effective_handler_types.update({HandlerType.Lstdout, HandlerType.Lstderr}) + + # Check if current logger has stream handlers, convert to handlertypes + existing_stream_handlers = { + HandlerType.Lstdout + if _logger_has_handler( + log, logging.StreamHandler, target_stream=cast(io.IOBase, sys.stdout) + ) + else None, + HandlerType.Lstderr + if _logger_has_handler( + log, logging.StreamHandler, target_stream=cast(io.IOBase, sys.stderr) + ) + else None, + } + existing_stream_handlers.discard(None) + print(f"{existing_stream_handlers=}") + + # Check if current logger has the interested handler + existing_handlers = { + HandlerType.Rich if _logger_has_handler(log, FormattedRichHandler) else None, + HandlerType.Protobufstream + if _logger_has_handler(log, ERSKafkaLogHandler) + else None, + HandlerType.Throttle if _logger_has_filter(log, ThrottleFilter) else None, + } + existing_handlers.discard(None) + existing_handlers.update(existing_stream_handlers) + + # print(f"{existing_handlers=}") + # print(f"{effective_handler_types=}") + + handlers_init_map = { + HandlerType.Rich: lambda: add_rich_handler(log, True, default_case), + HandlerType.Lstdout: lambda: add_stdout_handler(log, True, default_case), + HandlerType.Lstderr: lambda: add_stderr_handler(log, True, default_case), + HandlerType.Protobufstream: lambda: add_ers_kafka_handler( + log, True, ers_session_name, default_case + ), + HandlerType.Throttle: lambda: add_throttle_filter(log, {HandlerType.Rich}) + } + + supported_handers = [ + HandlerType.Rich, + HandlerType.Lstdout, + HandlerType.Lstderr, + HandlerType.Protobufstream, + HandlerType.Throttle, + ] + + for handler_type in supported_handers: + if handler_type not in effective_handler_types: + continue + if handler_type in existing_handlers: + continue + installer = handlers_init_map.get(handler_type) + if installer is None: + continue + print(handler_type) + + installer() + diff --git a/src/daqpytools/logging/logger.py b/src/daqpytools/logging/logger.py index 2f99d17..4b89186 100644 --- a/src/daqpytools/logging/logger.py +++ b/src/daqpytools/logging/logger.py @@ -8,12 +8,15 @@ from daqpytools.logging.exceptions import LoggerSetupError from daqpytools.logging.handlers import ( - ThrottleFilter, + LogHandlerConf, + add_handlers_from_types, + add_throttle_filter, add_ers_kafka_handler, add_file_handler, add_rich_handler, add_stderr_handler, add_stdout_handler, + HandlerType ) from daqpytools.logging.levels import logging_log_level_to_int from daqpytools.logging.utils import get_width @@ -71,8 +74,10 @@ def get_daq_logger( rich_handler: bool = False, file_handler_path: str | None = None, stream_handlers: bool = False, - ers_kafka_handler: bool = False, - throttle: bool = False + ers_kafka_handler: str | None = None, + throttle: bool = False, + + setup_ers_handlers: bool = False, ) -> logging.Logger: """C'tor for the default logging instances. @@ -84,7 +89,7 @@ def get_daq_logger( file_handler_path (str | None): Path to the file handler log file. If None, no file handler is added. stream_handlers (bool): Whether to add both stdout and stderr stream handlers. - ers_kafka_handler (bool): Whether to add an ERS protobuf handler. + ers_kafka_handler (str): Whether to add an ERS protobuf handler. str is session name throttle (bool): Whether to add the throttle filter or not. Note, does not mean outputs are filtered by default! See ThrottleFilter for details. @@ -141,20 +146,44 @@ def get_daq_logger( logger.setLevel(log_level) logger.propagate = use_parent_handlers + #! Okay so before this bit, you capture all the handlers that you want to have + # That would now be the default base handlers + # You apss this in each of the requested handlers here.. + + + default_case = {HandlerType.Rich} + # Add requested handlers + # if rich_handler: + # add_rich_handler(logger, use_parent_handlers) + # if file_handler_path: + # add_file_handler(logger, use_parent_handlers, file_handler_path) + # if stream_handlers: + # add_stdout_handler(logger, use_parent_handlers) + # add_stderr_handler(logger, use_parent_handlers) + # if ers_kafka_handler: + # add_ers_kafka_handler(logger, use_parent_handlers, ers_kafka_handler) + + # if throttle: + # # Note: Default parameters used. No functionality on customisability yet + # add_throttle_filter(logger) + + if rich_handler: - add_rich_handler(logger, use_parent_handlers) + add_rich_handler(logger, use_parent_handlers, default_case) if file_handler_path: - add_file_handler(logger, use_parent_handlers, file_handler_path) + add_file_handler(logger, use_parent_handlers, file_handler_path, default_case) if stream_handlers: - add_stdout_handler(logger, use_parent_handlers) - add_stderr_handler(logger, use_parent_handlers) - if ers_kafka_handler: - add_ers_kafka_handler(logger, use_parent_handlers, "session_tester") + add_stdout_handler(logger, use_parent_handlers, default_case) + add_stderr_handler(logger, use_parent_handlers, default_case) + if ers_kafka_handler: + add_ers_kafka_handler(logger, use_parent_handlers, ers_kafka_handler, default_case) if throttle: # Note: Default parameters used. No functionality on customisability yet - logger.addFilter(ThrottleFilter()) + add_throttle_filter(logger, default_case) + + # Set log level for all handlers if requested if log_level is not logging.NOTSET: @@ -166,3 +195,21 @@ def get_daq_logger( handler.setLevel(log_level) return logger + + +#! This will mean now that you need some function here that will allow you to go through all the handlers and all the filters and update that stupid self.default_case + +def setup_daq_ers_logger(logger, ers_session_name): + + # need to grab the list of relevant handlers that exist in ERS + #! This is very dependent on ERS env variables existing!!! + + all_handlers = {handler for handler_conf in LogHandlerConf._get_oks_conf().values() for handler in handler_conf.handlers} + + print(f"{all_handlers=}") + + add_handlers_from_types(logger, all_handlers, ers_session_name) + + + # now what.. Well we have a list of handlers to add now huh.. +