eris- Elasticsearch Recon Ingestion Scripts (ERIS) 🔎 |
git clone git://git.acid.vegas/-c.git |
Log | Files | Refs | Archive | README | LICENSE |
commit fd617e3c9d45ff29e8b6419ff0869213eeffee25
parent 90d6260b99b4395a4b0dcc0a72bd122fd1dd7e74 Author: acidvegas <acid.vegas@acid.vegas> Date: Wed, 6 Mar 2024 14:38:34 -0500 Certstream ingestion improved, still need to test stripping nulls and refactor the index mapping so we only store whats needed Diffstat:
|
1 file changed, 6 insertions(+), 4 deletions(-) |
diff --git a/ingestors/ingest_certs.py b/ingestors/ingest_certs.py @@ -101,26 +101,28 @@ async def process_data(file_path: str = None): while True: try: async with websockets.connect('wss://certstream.calidog.io/') as websocket: - while True: line = await websocket.recv() + if line == '~eof': # Sentinel value to indicate the end of a process (Used with --watch with FIFO) + break + try: record = json.loads(line) except json.decoder.JSONDecodeError: logging.error(f'Failed to parse JSON record from Certstream! ({line})') - input('Press Enter to continue...') # Pause the script to allow the user to read the error message + input('Press Enter to continue...') continue yield record except websockets.ConnectionClosed: logging.error('Connection to Certstream was closed. Attempting to reconnect...') - await asyncio.sleep(10) + await asyncio.sleep(15) except Exception as e: logging.error(f'An error occurred while processing Certstream records! ({e})') - await asyncio.sleep(10) + await asyncio.sleep(15) async def strip_struct_empty(data: dict) -> dict: |