eris

- Elasticsearch Recon Ingestion Scripts (ERIS) 🔎
git clone git://git.acid.vegas/-c.git
ingest_zone.py (4776B)
      1 #!/usr/bin/env python
      2 # Elasticsearch Recon Ingestion Scripts (ERIS) - Developed by Acidvegas (https://git.acid.vegas/eris)
      3 # ingest_zone.py
      4 
      5 import time
      6 
      7 default_index = 'dns-zones'
      8 record_types = ('a','aaaa','caa','cdnskey','cds','cname','dnskey','ds','mx','naptr','ns','nsec','nsec3','nsec3param','ptr','rrsig','rp','sshfp','soa','srv','txt','type65534')
      9 
     10 def construct_map() -> dict:
     11     '''Construct the Elasticsearch index mapping for zone file records.'''
     12 
     13     keyword_mapping = { 'type': 'text',  'fields': { 'keyword': { 'type': 'keyword', 'ignore_above': 256 } } }
     14 
     15     mapping = {
     16         'mappings': {
     17             'properties': {
     18                 'domain':  keyword_mapping,
     19                 'records': { 'properties': {} },
     20                 'seen':    {'type': 'date'}
     21             }
     22         }
     23     }
     24 
     25     # Add record types to mapping dynamically to not clutter the code
     26     for item in record_types:
     27         if item in ('a','aaaa'):
     28             mapping['mappings']['properties']['records']['properties'][item] = {
     29                 'properties': {
     30                     'data': { 'type': 'ip' },
     31                     'ttl':  { 'type': 'integer' }
     32                 }
     33             }
     34         else:
     35             mapping['mappings']['properties']['records']['properties'][item] = {
     36                 'properties': {
     37                 'data': keyword_mapping,
     38                 'ttl':  { 'type': 'integer' }
     39                 }
     40             }
     41 
     42     return mapping
     43 
     44 
     45 def process_file(file_path: str):
     46     '''
     47     Read and process zone file records.
     48 
     49     :param file_path: Path to the zone file
     50     '''
     51 
     52     domain_records = {}
     53     last_domain = None
     54 
     55     with open(file_path, 'r') as file:
     56         for line in file:
     57             line = line.strip()
     58 
     59             if not line or line.startswith(';'):
     60                 continue
     61 
     62             parts = line.split()
     63 
     64             if len(parts) < 5:
     65                 raise ValueError(f'Invalid line: {line}')
     66 
     67             domain, ttl, record_class, record_type, data = parts[0].rstrip('.').lower(), parts[1], parts[2].lower(), parts[3].lower(), ' '.join(parts[4:])
     68 
     69             if not ttl.isdigit():
     70                 raise ValueError(f'Invalid TTL: {ttl} with line: {line}')
     71             
     72             ttl = int(ttl)
     73 
     74             if record_class != 'in':
     75                 raise ValueError(f'Unsupported record class: {record_class} with line: {line}') # Anomaly (Doubtful any CHAOS/HESIOD records will be found)
     76 
     77             # We do not want to collide with our current mapping (Again, this is an anomaly)
     78             if record_type not in record_types:
     79                 raise ValueError(f'Unsupported record type: {record_type} with line: {line}')
     80 
     81             # Little tidying up for specific record types
     82             if record_type == 'nsec':
     83                 data = ' '.join([data.split()[0].rstrip('.'), *data.split()[1:]])
     84             elif record_type == 'soa':
     85                     data = ' '.join([part.rstrip('.') if '.' in part else part for part in data.split()])
     86             elif data.endswith('.'):
     87                 data = data.rstrip('.')
     88 
     89             if domain != last_domain:
     90                 if last_domain:
     91                     source = {'domain': last_domain, 'records': domain_records[last_domain], 'seen': time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime())}
     92                     
     93                     del domain_records[last_domain]
     94 
     95                     yield source
     96 
     97                 last_domain = domain
     98 
     99                 domain_records[domain] = {}
    100 
    101             if record_type not in domain_records[domain]:
    102                 domain_records[domain][record_type] = []
    103 
    104             domain_records[domain][record_type].append({'ttl': ttl, 'data': data})
    105 
    106     return None # EOF
    107 
    108 
    109 
    110 '''
    111 Example record:
    112 0so9l9nrl425q3tf7dkv1nmv2r3is6vm.vegas. 3600    in  nsec3   1 1 100 332539EE7F95C32A 10MHUKG4FHIAVEFDOTF6NKU5KFCB2J3A NS DS RRSIG
    113 0so9l9nrl425q3tf7dkv1nmv2r3is6vm.vegas. 3600    in  rrsig   NSEC3 8 2 3600 20240122151947 20240101141947 4125 vegas. hzIvQrZIxBSwRWyiHkb5M2W0R3ikNehv884nilkvTt9DaJSDzDUrCtqwQb3jh6+BesByBqfMQK+L2n9c//ZSmD5/iPqxmTPCuYIB9uBV2qSNSNXxCY7uUt5w7hKUS68SLwOSjaQ8GRME9WQJhY6gck0f8TT24enjXXRnQC8QitY=
    114 1-800-flowers.vegas.    3600    in  ns  dns1.cscdns.net.
    115 1-800-flowers.vegas.    3600    in  ns  dns2.cscdns.net.
    116 100.vegas.  3600    in  ns  ns51.domaincontrol.com.
    117 100.vegas.  3600    in  ns  ns52.domaincontrol.com.
    118 1001.vegas. 3600    in  ns  ns11.waterrockdigital.com.
    119 1001.vegas. 3600    in  ns  ns12.waterrockdigital.com.
    120 
    121 Will be indexed as:
    122 {
    123     "domain": "1001.vegas",
    124     "records": {
    125         "ns": [
    126             {"ttl": 3600, "data": "ns11.waterrockdigital.com"},
    127             {"ttl": 3600, "data": "ns12.waterrockdigital.com"}
    128         ]
    129     },
    130     "seen": "2021-09-01T00:00:00Z" # Zulu time added upon indexing
    131 }
    132 '''