eris

- Elasticsearch Recon Ingestion Scripts (ERIS) 🔎
git clone git://git.acid.vegas/eris.git
ingest_rir_whois.py (5975B)
      1 #/usr/bin/env python
      2 # opengeoip - developed by acidvegas in python (https://git.acid.vegas/opengeoip)
      3 # whois.py
      4 
      5 '''
      6 Source: https://ftp.[afrinic|apnic|arin|lacnic|ripe].net/ + others
      7 Notes : The databases are updated daily.
      8 		Check out the whois2json.py script for converting the whois data into JSON objects for elasticsearch.
      9 		Each database has a CURRENTSERIAL file that contains the serial number of the latest version of the database.
     10 		Need to find a smart way to unify the approach for each RIR to account for their differences.
     11 		Do we need ripe-nonauth.db.gz?      | https://ftp.ripe.net/ripe/dbase/
     12 		Do we need twnic, krnic, jpnic?     | https://ftp.apnic.net/apnic/dbase/data/
     13 		Do we need the IRR databases?       | https://www.irr.net/docs/list.html
     14 		Do we need the LACNIC IRR database? | https://ftp.lacnic.net/lacnic/irr/
     15 '''
     16 
     17 import os
     18 import sys
     19 
     20 try:
     21     import utils
     22 except ImportError:
     23     raise SystemError('utils.py module not found!')
     24 
     25 REGISTRY_DATA = {
     26 	'AFRINIC'  : 'ftp.afrinic.net/dbase/afrinic.db.gz',
     27 	'APNIC'    : ['ftp.apnic.net/apnic/whois/apnic.db.as-block.gz', # APNIC really has to be "special" snowflake...
     28 				 'ftp.apnic.net/apnic/whois/apnic.db.as-set.gz',
     29 				 'ftp.apnic.net/apnic/whois/apnic.db.aut-num.gz',
     30 				 'ftp.apnic.net/apnic/whois/apnic.db.domain.gz',
     31 				 'ftp.apnic.net/apnic/whois/apnic.db.filter-set.gz',
     32 				 'ftp.apnic.net/apnic/whois/apnic.db.inet-rtr.gz',
     33 				 'ftp.apnic.net/apnic/whois/apnic.db.inet6num.gz',
     34 				 'ftp.apnic.net/apnic/whois/apnic.db.inetnum.gz',
     35 				 'ftp.apnic.net/apnic/whois/apnic.db.irt.gz',
     36 				 'ftp.apnic.net/apnic/whois/apnic.db.key-cert.gz',
     37 				 'ftp.apnic.net/apnic/whois/apnic.db.limerick.gz', # Little easteregg, not updated since 2017...
     38 				 'ftp.apnic.net/apnic/whois/apnic.db.mntner.gz',
     39 				 'ftp.apnic.net/apnic/whois/apnic.db.organisation.gz',
     40 				 'ftp.apnic.net/apnic/whois/apnic.db.peering-set.gz',
     41 				 'ftp.apnic.net/apnic/whois/apnic.db.role.gz',
     42 				 'ftp.apnic.net/apnic/whois/apnic.db.route-set.gz',
     43 				 'ftp.apnic.net/apnic/whois/apnic.db.route.gz',
     44 				 'ftp.apnic.net/apnic/whois/apnic.db.route6.gz',
     45 				 'ftp.apnic.net/apnic/whois/apnic.db.rtr-set.gz'],
     46 	'JPNIC'    : 'ftp.apnic.net/apnic/dbase/data/jpnic.db.gz', # JPNIC  is APNIC but is a NIR
     47 	'KRNIC'    : 'ftp.apnic.net/apnic/dbase/data/krnic.db.gz', # KRNIC  is APNIC but is a NIR
     48 	'TWINIC'   : 'ftp.apnic.net/apnic/dbase/data/twnic.db.gz', # TWINIC is APNIC but is a NIR
     49 	'ARIN'     : 'ftp.arin.net/pub/rr/arin.db.gz',
     50 	'LACNIC'   : ['ftp.lacnic.net/lacnic/dbase/lacnic.db.gz', # LACNIC has a dbase and an IRR
     51 				 'ftp.lacnic.net/lacnic/irr/lacnic.db.gz'],
     52 	'RIPE'     : ['ftp.ripe.net/ripe/dbase/ripe.db.gz', # RIPE has a dbase and a non-auth dbase
     53 				 'ftp.ripe.net/ripe/dbase/ripe-nonauth.db.gz']
     54 }
     55 
     56 IRR_DATA = {
     57 	'ALTDB'    : 'ftp.altdb.net/pub/altdb/altdb.db.gz',
     58 	'BELL'     : 'whois.in.bell.ca/bell.db.gz',
     59 	'CANARIE'  : 'ftp.radb.net/radb/dbase/canarie.db.gz',
     60 	'HOST'     : 'ftp.radb.net/radb/dbase/host.db.gz',
     61 	'IDNIC'    : 'irr-mirror.idnic.net/idnic.db.gz',
     62 	'JPIRR'    : 'ftp.nic.ad.jp/jpirr/jpirr.db.gz',
     63 	'LACNIC'   : 'ftp.lacnic.net/lacnic/irr/lacnic.db.gz', # Name conflict...
     64 	'LEVEL3'   : 'rr.Level3.net/level3.db.gz',
     65 	'NESTEGG'  : 'ftp.nestegg.net/irr/nestegg.db.gz',
     66 	'NTTCOM'   : 'rr1.ntt.net/nttcomRR/nttcom.db.gz',
     67 	'OPENFACE' : 'ftp.radb.net/radb/dbase/openface.db.gz',
     68 	'PANIX'    : 'ftp.panix.com/pub/rrdb/panix.db.gz',
     69 	'RADB'     : 'ftp.radb.net/radb/dbase/radb.db.gz',
     70 	'REACH'    : 'ftp.radb.net/radb/dbase/reach.db.gz',
     71 	'RGNET'    : 'ftp.radb.net/radb/dbase/rgnet.db.gz',
     72 	'TC'       : 'ftp.bgp.net.br/tc.db.gz'
     73 }
     74 
     75 
     76 def check_serial(local_serial_path: str, ftp_serial_url: str) -> bool:
     77 	'''
     78 	Check the CURRENTSERIAL file for an update.
     79 
     80 	:param local_path: The path to the local CURRENTSERIAL file
     81 	:param ftp_path: The path to the remote CURRENTSERIAL file
     82 	'''
     83 
     84 	if not os.path.exists(local_serial_path):
     85 		return True
     86 
     87 	else:
     88 		with open(local_serial_path) as serial_file:
     89 			old_serial = serial_file.read().strip()
     90 		new_serial = utils.read_ftp(ftp_serial_url)
     91 		if old_serial != new_serial:
     92 			return True
     93 		else:
     94 			return False
     95 
     96 
     97 def download_db(output_path: str, url: str):
     98 	'''
     99 	Dowload a WHOIS database.
    100 
    101 	:param output_path: The path to the output directory
    102 	:param url: The FTP URL of the database to download
    103 	'''
    104 
    105 	DB_PATH = os.path.join(output_path, url.split('/')[-1])
    106 	utils.safe_remove(DB_PATH)
    107 	utils.safe_remove(DB_PATH[:-3])
    108 	utils.download_ftp(url, DB_PATH)
    109 	utils.gunzip_extract(DB_PATH, DB_PATH[:-3])
    110 	utils.safe_remove(DB_PATH)
    111 
    112 
    113 def update_db(output_path: str):
    114 	'''
    115 	Update the whois IRR databases.
    116 
    117 	:param output_path: The path to the output directory
    118 	'''
    119 
    120 	os.makedirs(output_path, exist_ok=True)
    121 
    122 	for RIR, DB_URL in REGISTRY_DATA.items():
    123 		if RIR == 'APNIC':
    124 			for url in DB_URL:
    125 				download_db(output_path, url)
    126 		else:
    127 			download_db(output_path, DB_URL)
    128 	for IRR, DB_URL in REGISTRY_DATA.items():
    129 		download_db(output_path+'/irr', DB_URL)
    130 
    131 		# For now, we'll just download the files and not check the serials for updates APNIC will need to be handled differently...seperate function?
    132 		'''
    133 		SERIAL_PATH = os.path.join(output_path, REGISTRY+'.CURRENTSERIAL')
    134 		FTP_SERIAL  = os.path.join('/'.join(DB_URL.split('/')[:-1]), REGISTRY + '.CURRENTSERIAL')
    135 
    136 		if check_serial(SERIAL_PATH, FTP_SERIAL):
    137 			logging.debug(f'A new version of the {REGISTRY} whois database is available! Downloading...')			
    138 			for item in (DB_PATH, DB_PATH[:-3], SERIAL_PATH):
    139 				utils.safe_remove(item) # Remove old files
    140 			utils.download_ftp(DB_URL, DB_PATH)
    141 			utils.download_ftp(FTP_SERIAL, SERIAL_PATH)
    142 			utils.gunzip_extract(DB_PATH, DB_PATH[:-3])
    143 			logging.debug(f'The {REGISTRY} whois database has been updated!')
    144 		else:
    145 			logging.info(f'The {REGISTRY} whois database is up-to-date')
    146 		'''
    147 
    148 
    149 if __name__ == '__main__':
    150 	utils.print_header('WHOIS Module')
    151 	utils.setup_logger('whois')
    152 	utils.setup_user_agent()
    153 	update_db('database/whois')