eris

- Elasticsearch Recon Ingestion Scripts (ERIS) 🔎
git clone git://git.acid.vegas/-c.git
ingest_certs.py (4656B)
      1 #!/usr/bin/env python
      2 # Elasticsearch Recon Ingestion Scripts (ERIS) - Developed by Acidvegas (https://git.acid.vegas/eris)
      3 # ingest_certs.py
      4 
      5 import asyncio
      6 import json
      7 import logging
      8 import time
      9 
     10 try:
     11 	import websockets
     12 except ImportError:
     13 	raise ImportError('Missing required \'websockets\' library. (pip install websockets)')
     14 
     15 
     16 # Set a default elasticsearch index if one is not provided
     17 default_index = 'cert-stream'
     18 
     19 
     20 def construct_map() -> dict:
     21 	'''Construct the Elasticsearch index mapping for Certstream records.'''
     22 
     23 	# Match on exact value or full text search
     24 	keyword_mapping = { 'type': 'text', 'fields': { 'keyword': { 'type': 'keyword', 'ignore_above': 256 } } }
     25 
     26 	# Construct the index mapping
     27 	mapping = {
     28 		'mappings': {
     29 			'properties' : {
     30 				'domain' : keyword_mapping,
     31 				'seen'   : { 'type': 'date' }
     32 			}
     33 		}
     34 	}
     35 
     36 	return mapping
     37 
     38 
     39 async def process_data(place_holder: str = None):
     40 	'''
     41 	Read and process Certsream records live from the Websocket stream.
     42 
     43 	:param place_holder: Placeholder parameter to match the process_data function signature of other ingestors.
     44 	'''
     45 
     46 	while True:
     47 		try:
     48 			async with websockets.connect('wss://certstream.calidog.io') as websocket:
     49 				while True:
     50 					# Read a line from the websocket
     51 					line = await websocket.recv()
     52 
     53 					# Parse the JSON record
     54 					try:
     55 						record = json.loads(line)
     56 					except json.decoder.JSONDecodeError:
     57 						logging.error(f'Invalid line from the websocket: {line}')
     58 						continue
     59 
     60 					# Grab the unique domains from the record (excluding wildcards)
     61 					domains = record['data']['leaf_cert']['all_domains']
     62 					domains = set([domain[2:] if domain.startswith('*.') else domain for domain in domains])
     63 
     64 					# Construct the document
     65 					for domain in domains:
     66 						struct = {
     67 							'domain' : domain,
     68 							'seen'   : time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime())
     69 						}
     70 
     71 						yield {'_id': id, '_index': default_index, '_source': struct}
     72 
     73 		except websockets.ConnectionClosed:
     74 			logging.error('Connection to Certstream was closed. Attempting to reconnect...')
     75 			await asyncio.sleep(15)
     76 
     77 		except Exception as e:
     78 			logging.error(f'An error occurred while processing Certstream records! ({e})')
     79 			break
     80 
     81 
     82 async def test():
     83 	'''Test the ingestion process.'''
     84 
     85 	async for document in process_data():
     86 		print(document)
     87 
     88 
     89 
     90 if __name__ == '__main__':
     91 	import asyncio
     92 
     93 	asyncio.run(test())
     94 
     95 
     96 
     97 '''
     98 Output:
     99 	{
    100 		"data": {
    101 			"cert_index": 43061646,
    102 			"cert_link": "https://yeti2025.ct.digicert.com/log/ct/v1/get-entries?start=43061646&end=43061646",
    103 			"leaf_cert": {
    104 				"all_domains": [
    105 					"*.d7zdnegbre53n.amplifyapp.com",
    106 					"d7zdnegbre53n.amplifyapp.com"
    107 				],
    108 				"extensions": {
    109 					"authorityInfoAccess"    : "CA Issuers - URI:http://crt.r2m02.amazontrust.com/r2m02.cer\nOCSP - URI:http://ocsp.r2m02.amazontrust.com\n",
    110 					"authorityKeyIdentifier" : "keyid:C0:31:52:CD:5A:50:C3:82:7C:74:71:CE:CB:E9:9C:F9:7A:EB:82:E2\n",
    111 					"basicConstraints"       : "CA:FALSE",
    112 					"certificatePolicies"    : "Policy: 2.23.140.1.2.1",
    113 					"crlDistributionPoints"  : "Full Name:\n URI:http://crl.r2m02.amazontrust.com/r2m02.crl",
    114 					"ctlPoisonByte"          : true,
    115 					"extendedKeyUsage"       : "TLS Web server authentication, TLS Web client authentication",
    116 					"keyUsage"               : "Digital Signature, Key Encipherment",
    117 					"subjectAltName"         : "DNS:d7zdnegbre53n.amplifyapp.com, DNS:*.d7zdnegbre53n.amplifyapp.com",
    118 					"subjectKeyIdentifier"   : "59:32:78:2A:11:03:62:55:BB:3B:B9:80:24:76:28:90:2E:D1:A4:56"
    119 				},
    120 				"fingerprint": "D9:05:A3:D5:AA:F9:68:BC:0C:0A:15:69:C9:5E:11:92:32:67:4F:FA",
    121 				"issuer": {
    122 					"C"            : "US",
    123 					"CN"           : "Amazon RSA 2048 M02",
    124 					"L"            : null,
    125 					"O"            : "Amazon",
    126 					"OU"           : null,
    127 					"ST"           : null,
    128 					"aggregated"   : "/C=US/CN=Amazon RSA 2048 M02/O=Amazon",
    129 					"emailAddress" : null
    130 				},
    131 				"not_after"           : 1743811199,
    132 				"not_before"          : 1709596800,
    133 				"serial_number"       : "FDB450C1942E3D30A18737063449E62",
    134 				"signature_algorithm" : "sha256, rsa",
    135 				"subject": {
    136 					"C"            : null,
    137 					"CN"           : "*.d7zdnegbre53n.amplifyapp.com",
    138 					"L"            : null,
    139 					"O"            : null,
    140 					"OU"           : null,
    141 					"ST"           : null,
    142 					"aggregated"   : "/CN=*.d7zdnegbre53n.amplifyapp.com",
    143 					"emailAddress" : null
    144 				}
    145 			},
    146 			"seen": 1709651773.594684,
    147 			"source": {
    148 				"name" : "DigiCert Yeti2025 Log",
    149 				"url"  : "https://yeti2025.ct.digicert.com/log/"
    150 			},
    151 			"update_type": "PrecertLogEntry"
    152 		},
    153 		"message_type": "certificate_update"
    154 	}
    155 '''