eris

- Elasticsearch Recon Ingestion Scripts (ERIS) 🔎
git clone git://git.acid.vegas/-c.git
ingest_masscan.py (4826B)
      1 #!/usr/bin/env python
      2 # Elasticsearch Recon Ingestion Scripts (ERIS) - Developed by Acidvegas (https://git.acid.vegas/eris)
      3 # ingest_masscan.py
      4 
      5 '''
      6 apt-get install iptables masscan libpcap-dev screen
      7 setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip' /bin/masscan
      8 /sbin/iptables -A INPUT -p tcp --dport 61010 -j DROP
      9 printf "0.0.0.0/8\n10.0.0.0/8\n100.64.0.0/10\n127.0.0.0/8\n169.254.0.0/16\n172.16.0.0/12\n192.0.0.0/24\n192.0.2.0/24\n192.31.196.0/24\n192.52.193.0/24\n192.88.99.0/24\n192.168.0.0/16\n192.175.48.0/24\n198.18.0.0/15\n198.51.100.0/24\n203.0.113.0/24\n224.0.0.0/3\n255.255.255.255/32"  > exclude.conf
     10 screen -S scan
     11 masscan 0.0.0.0/0 -p21,22,23 --banners --http-user-agent "USER_AGENT" --source-port 61010 --open-only --rate 30000 --excludefile exclude.conf -oJ output.json
     12 masscan 0.0.0.0/0 -p21,22,23 --banners --http-user-agent "USER_AGENT" --source-port 61000-65503 --open-only --rate 30000 --excludefile exclude.conf -oJ output_new.json --shard $i/$TOTAL
     13 
     14 Note: The above iptables rule is not persistent and will be removed on reboot.
     15 '''
     16 
     17 import json
     18 import logging
     19 import re
     20 import time
     21 
     22 default_index = 'masscan-logs'
     23 
     24 def construct_map() -> dict:
     25     '''Construct the Elasticsearch index mapping for Masscan records.'''
     26 
     27     keyword_mapping = { 'type': 'text',  'fields': { 'keyword': { 'type': 'keyword', 'ignore_above': 256 } } }
     28 
     29     mapping = {
     30         'mappings': {
     31             'properties': {
     32                 'ip':      { 'type': 'ip' },
     33                 'port':    { 'type': 'integer' },
     34                 'proto':   { 'type': 'keyword' },
     35                 'service': { 'type': 'keyword' },
     36                 'banner':  keyword_mapping,
     37                 'ref_id':  { 'type': 'keyword' },
     38                 'seen':    { 'type': 'date' }
     39                 #'geoip':   {
     40                 #    'properties': {
     41                 #        'city_name':        keyword_mapping,
     42                 #        'continent_name':   keyword_mapping,
     43                 #        'country_iso_code': keyword_mapping,
     44                 #        'country_name':     keyword_mapping,
     45                 #        'location':         { 'type': 'geo_point' },
     46                 #        'region_iso_code':  keyword_mapping,
     47                 #        'region_name':      keyword_mapping,
     48                 #    }
     49                 #}
     50             }
     51         }
     52     }
     53 
     54     return mapping
     55 
     56 
     57 def process_file(file_path: str):
     58     '''
     59     Read and process Masscan records from the log file.
     60 
     61     :param file_path: Path to the Masscan log file
     62     '''
     63 
     64     with open(file_path, 'r') as file:
     65         for line in file:
     66             line = line.strip()
     67 
     68             if not line or not line.startswith('{'):
     69                 continue
     70 
     71             if line.endswith(','):
     72                 line = line[:-1]
     73 
     74             try:
     75                 record = json.loads(line)
     76             except json.decoder.JSONDecodeError:
     77                 logging.error(f'Failed to parse JSON record! ({line})')
     78                 input('Press Enter to continue...') # Debugging
     79                 continue
     80 
     81             for port_info in record['ports']:
     82                 struct = {
     83                     'ip': record['ip'],
     84                     'port': port_info['port'],
     85                     'proto': port_info['proto'],
     86                     'seen': time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime(int(record['timestamp']))),
     87                 }
     88 
     89                 if 'service' in port_info:
     90                     if 'name' in port_info['service']:
     91                         if port_info['service']['name'] != 'unknown':
     92                             struct['service'] = port_info['service']['name']
     93 
     94                     if 'banner' in port_info['service']:
     95                         banner = ' '.join(port_info['service']['banner'].split()) # Remove extra whitespace
     96                         if banner:
     97                             match = re.search(r'\(Ref\.Id: (.*?)\)', banner)
     98                             if match:
     99                                 struct['ref_id'] = match.group(1)
    100                             else:
    101                                 struct['banner'] = banner
    102 
    103                 yield struct
    104  
    105     return None # EOF
    106 
    107 
    108 
    109 '''
    110 Example record:
    111 {
    112     "ip": "43.134.51.142",
    113     "timestamp": "1705255468", # Convert to ZULU BABY
    114     "ports": [ # We will create a record for each port opened
    115         {
    116             "port": 22,
    117             "proto": "tcp",
    118             "service": { # This field is optional
    119                 "name": "ssh",
    120                 "banner": "SSH-2.0-OpenSSH_8.9p1 Ubuntu-3ubuntu0.4"
    121             }
    122         }
    123     ]
    124 }
    125 
    126 Will be indexed as:
    127 {
    128     "ip": "43.134.51.142",
    129     "port": 22,
    130     "proto": "tcp",
    131     "service": "ssh",
    132     "banner": "SSH-2.0-OpenSSH_8.9p1 Ubuntu-3ubuntu0.4",
    133     "seen": "2021-10-08T02:04:28Z",
    134     "ref_id": "?sKfOvsC4M4a2W8PaC4zF?" # TCP RST Payload (Do we need this?)
    135 }
    136 '''