eris

- Elasticsearch Recon Ingestion Scripts (ERIS) 🔎
git clone git://git.acid.vegas/-c.git
ingest_massdns.py (2607B)
      1 #!/usr/bin/env python
      2 # Elasticsearch Recon Ingestion Scripts (ERIS) - Developed by Acidvegas (https://git.acid.vegas/eris)
      3 # ingest_massdns.py
      4 
      5 import time
      6 
      7 try:
      8     import aiofiles
      9 except ImportError:
     10     raise ImportError('Missing required \'aiofiles\' library. (pip install aiofiles)')
     11 
     12 default_index = 'ptr-records'
     13 
     14 def construct_map() -> dict:
     15     '''Construct the Elasticsearch index mapping for MassDNS records'''
     16 
     17     keyword_mapping = { 'type': 'text',  'fields': { 'keyword': { 'type': 'keyword', 'ignore_above': 256 } } }
     18 
     19     mapping = {
     20     'mappings': {
     21             'properties': {
     22                 'ip':     { 'type': 'ip' },
     23                 'name':   { 'type': 'keyword' },
     24                 'record': keyword_mapping,
     25                 'seen':   { 'type': 'date' }
     26             }
     27         }
     28     }
     29 
     30     return mapping
     31 
     32 
     33 async def process_data(file_path: str):
     34     '''
     35     Read and process Massdns records from the log file.
     36 
     37     :param file_path: Path to the Massdns log file
     38     '''
     39 
     40     async with aiofiles.open(file_path, mode='r') as input_file:
     41         async for line in input_file:
     42             line = line.strip()
     43 
     44             if not line:
     45                 continue
     46 
     47             parts = line.split()
     48 
     49             if len(parts) < 3:
     50                 raise ValueError(f'Invalid PTR record: {line}')
     51             
     52             name, record_type, data = parts[0].rstrip('.'), parts[1], ' '.join(parts[2:]).rstrip('.')
     53 
     54             if record_type != 'PTR':
     55                 continue
     56 
     57                 #if record_type == 'CNAME':
     58                 #    if data.endswith('.in-addr.arpa'):
     59                 #        continue
     60 
     61             # Let's not index the PTR record if it's the same as the in-addr.arpa domain
     62             if data == name:
     63                 continue
     64                     
     65             ip = '.'.join(name.replace('.in-addr.arpa', '').split('.')[::-1])
     66             
     67             struct = {
     68                 'ip': ip,
     69                 'record': data,
     70                 'seen': time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime())
     71             }
     72 
     73             yield {'_index': default_index, '_source': struct}
     74     
     75     return None # EOF
     76 
     77 
     78 '''
     79 Example PTR record:
     80 0.6.229.47.in-addr.arpa. PTR 047-229-006-000.res.spectrum.com.
     81 0.6.228.75.in-addr.arpa. PTR 0.sub-75-228-6.myvzw.com.
     82 0.6.207.73.in-addr.arpa. PTR c-73-207-6-0.hsd1.ga.comcast.net.
     83 0.6.212.173.in-addr.arpa. PTR 173-212-6-0.cpe.surry.net.
     84 0.6.201.133.in-addr.arpa. PTR flh2-133-201-6-0.tky.mesh.ad.jp.
     85 
     86 Will be indexed as:
     87 {
     88     "ip": "47.229.6.0",
     89     "record": "047-229-006-000.res.spectrum.com.",
     90     "seen": "2021-06-30T18:31:00Z"
     91 }
     92 '''