117 lines
4.0 KiB
Python
117 lines
4.0 KiB
Python
import os
|
||
|
||
from Infrostructure.ProtocolCoder.MessageEncoder import MessageEncoder
|
||
from Infrostructure.RabbitMQ.RabbitMQMessenger import RabbitMQMessenger
|
||
from Processor.StreamingLogCluster import StreamingLogCluster
|
||
|
||
|
||
class LogProcessingWorker:
|
||
def __init__(self,
|
||
model_path: str,
|
||
db_path: str,
|
||
input_queue: str = 'logs_input',
|
||
output_queue: str = 'logs_output',
|
||
output_debug_queue: str = 'logs_debug_output',):
|
||
|
||
if os.path.exists(db_path):
|
||
os.remove(db_path)
|
||
|
||
self.output_queue = output_queue
|
||
self.output_debug_queue = output_debug_queue
|
||
|
||
print("--- ЗАПУСК основоного алгоритма ---")
|
||
self.clusterer = StreamingLogCluster(model_path, db_path=db_path)
|
||
|
||
print("--- ЗАПУСК системы кодирования ---")
|
||
self.encoder = MessageEncoder()
|
||
|
||
print("--- ЗАПУСК системы приёма/отправки сообщений ---")
|
||
self.messenger = RabbitMQMessenger()
|
||
|
||
print("--- ЗАПУСК системы чтения сообщений ---")
|
||
self.messenger.start_listening(
|
||
queue_name=input_queue,
|
||
callback_function=self._process_log_callback
|
||
)
|
||
|
||
def _process_log_callback(self, log_text: str):
|
||
try:
|
||
log_text = log_text.strip()
|
||
if not log_text:
|
||
return
|
||
|
||
print(f" [>] Обработка: {log_text[:50]}...")
|
||
|
||
# А. Кластеризация
|
||
# process() возвращает dict, который полностью готов к JSON
|
||
analysis_result = self.clusterer.process(log_text)
|
||
|
||
me = MessageEncoder()
|
||
|
||
data = me.encode_protocol(analysis_result['template_id'],
|
||
[(i['uid'], i['value']) for i in analysis_result['variables']]
|
||
)
|
||
|
||
# Г. Отправка результата в Output очередь
|
||
# Messenger сам переподключится, если связь мигнула
|
||
self.messenger.send_binary_message(self.output_queue, data )
|
||
self.messenger.send_message(self.output_debug_queue, str(analysis_result))
|
||
|
||
except Exception as e:
|
||
print(f" [!] Ошибка внутри логики обработки: {e}")
|
||
|
||
|
||
def local_test():
|
||
MODEL_PATH = './Resources/model'
|
||
DB_FILE = "logs.db"
|
||
TEST_FILE = "./Resources/test/container-qfdpbp.log"
|
||
|
||
if os.path.exists(DB_FILE):
|
||
os.remove(DB_FILE)
|
||
|
||
print("--- ЗАПУСК основоного алгоритма ---")
|
||
clusterer = StreamingLogCluster(MODEL_PATH, db_path=DB_FILE)
|
||
|
||
print("--- ЗАПУСК системы кодирования ---")
|
||
encoder = MessageEncoder()
|
||
|
||
me = MessageEncoder()
|
||
|
||
new_len = 0
|
||
|
||
dict = {}
|
||
|
||
with open(TEST_FILE, 'r', errors='ignore') as f:
|
||
while True:
|
||
log_text = f.readline()
|
||
|
||
if log_text == "":
|
||
break
|
||
analysis_result = clusterer.process(log_text)
|
||
|
||
data = me.encode_protocol(analysis_result['template_id'],
|
||
[(i['uid'], i['value']) for i in analysis_result['variables']]
|
||
)
|
||
new_len += len(data)
|
||
|
||
if analysis_result['template_id'] in dict:
|
||
dict[analysis_result['template_id']] +=1
|
||
else:
|
||
dict[analysis_result['template_id']] = 1
|
||
print(f"[{len(data)}]->({analysis_result['template_id']})",data)
|
||
|
||
print(new_len / 1024)
|
||
print(dict,sep="\n")
|
||
|
||
if __name__ == '__main__':
|
||
local_test()
|
||
# MODEL_PATH = './Resources/model'
|
||
# DB_FILE = "logs.db"
|
||
# INPUT_QUEUE = "input"
|
||
# OUTPUT_QUEUE = "output"
|
||
# OUTPUT_DEBUG_QUEUE = "debug_output"
|
||
#
|
||
# processor = LogProcessingWorker(MODEL_PATH, DB_FILE, INPUT_QUEUE, OUTPUT_QUEUE, OUTPUT_DEBUG_QUEUE)
|
||
|
||
|