eris- Elasticsearch Recon Ingestion Scripts (ERIS) 🔎 |
git clone git://git.acid.vegas/-c.git |
Log | Files | Refs | Archive | README | LICENSE |
commit 1ab7199f7db2c66b08f9dba4743dbb48f15663ce
parent 7f93a4d8de7da1c603a1ee077c4edd5e6780dac0 Author: acidvegas <acid.vegas@acid.vegas> Date: Wed, 13 Mar 2024 22:34:20 -0400 Certstream ingestor now only logs sub-domains since we already ingested zone files. Ignores www. and wildcard domains. Diffstat:
|
1 file changed, 14 insertions(+), 4 deletions(-) |
diff --git a/ingestors/ingest_certstream.py b/ingestors/ingest_certstream.py @@ -54,10 +54,20 @@ async def process_data(place_holder: str = None): logging.error(f'Invalid line from the websocket: {line}') continue - # Grab the unique domains from the record (excluding wildcards) - domains = record['data']['leaf_cert']['all_domains'] - domains = set([domain[2:] if domain.startswith('*.') else domain for domain in domains]) - + # Grab the unique domains from the records + all_domains = record['data']['leaf_cert']['all_domains'] + domains = list() + + # We only care about subdomains (excluding www. and wildcards) + for domain in all_domains: + if domain.startswith('*.'): + domain = domain[2:] + elif domain.startswith('www.') and domain.count('.') == 2: + continue + if domain.count('.') > 1: + if domain not in domains: + domains.append(domain) + # Construct the document for domain in domains: struct = { |