diff --git a/ingestors/ingest_massdns.py b/ingestors/ingest_massdns.py
@@ -2,6 +2,7 @@
# Elasticsearch Recon Ingestion Scripts (ERIS) - Developed by Acidvegas (https://git.acid.vegas/eris)
# ingest_massdns.py
+import logging
import time
try:
@@ -17,15 +18,15 @@ def construct_map() -> dict:
keyword_mapping = { 'type': 'text', 'fields': { 'keyword': { 'type': 'keyword', 'ignore_above': 256 } } }
mapping = {
- 'mappings': {
- 'properties': {
- 'ip': { 'type': 'ip' },
- 'name': { 'type': 'keyword' },
- 'record': keyword_mapping,
- 'seen': { 'type': 'date' }
+ 'mappings': {
+ 'properties': {
+ 'ip' : { 'type' : 'ip' },
+ 'name' : { 'type' : 'keyword' },
+ 'record' : keyword_mapping,
+ 'seen' : { 'type' : 'date' }
+ }
}
}
- }
return mapping
@@ -41,6 +42,9 @@ async def process_data(file_path: str):
async for line in input_file:
line = line.strip()
+ if line == '~eof': # Sentinel value to indicate the end of a process (Used with --watch with FIFO)
+ break
+
if not line:
continue
@@ -49,44 +53,47 @@ async def process_data(file_path: str):
if len(parts) < 3:
raise ValueError(f'Invalid PTR record: {line}')
- name, record_type, data = parts[0].rstrip('.'), parts[1], ' '.join(parts[2:]).rstrip('.')
+ name, record_type, record = parts[0].rstrip('.'), parts[1], ' '.join(parts[2:]).rstrip('.')
+ # Do we handle CNAME records returned by MassDNS?
if record_type != 'PTR':
continue
- #if record_type == 'CNAME':
- # if data.endswith('.in-addr.arpa'):
- # continue
-
# Let's not index the PTR record if it's the same as the in-addr.arpa domain
- if data == name:
+ if record == name:
+ continue
+
+ if not record: # Skip empty records
continue
ip = '.'.join(name.replace('.in-addr.arpa', '').split('.')[::-1])
struct = {
- 'ip': ip,
- 'record': data,
- 'seen': time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime())
+ 'ip' : ip,
+ 'record' : record,
+ 'seen' : time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime())
}
- yield {'_index': default_index, '_source': struct}
-
- return None # EOF
+ yield {'_id': ip, '_index': default_index, '_source': struct} # Store with ip as the unique id to allow the record to be reindexed if it exists.
+
'''
Example PTR record:
-0.6.229.47.in-addr.arpa. PTR 047-229-006-000.res.spectrum.com.
-0.6.228.75.in-addr.arpa. PTR 0.sub-75-228-6.myvzw.com.
-0.6.207.73.in-addr.arpa. PTR c-73-207-6-0.hsd1.ga.comcast.net.
+0.6.229.47.in-addr.arpa. PTR 047-229-006-000.res.spectrum.com.
+0.6.228.75.in-addr.arpa. PTR 0.sub-75-228-6.myvzw.com.
+0.6.207.73.in-addr.arpa. PTR c-73-207-6-0.hsd1.ga.comcast.net.
0.6.212.173.in-addr.arpa. PTR 173-212-6-0.cpe.surry.net.
0.6.201.133.in-addr.arpa. PTR flh2-133-201-6-0.tky.mesh.ad.jp.
Will be indexed as:
{
- "ip": "47.229.6.0",
- "record": "047-229-006-000.res.spectrum.com.",
- "seen": "2021-06-30T18:31:00Z"
+ "_id" : "47.229.6.0"
+ "_index" : "ptr-records",
+ "_source" : {
+ "ip" : "47.229.6.0",
+ "record" : "047-229-006-000.res.spectrum.com.",
+ "seen" : "2021-06-30T18:31:00Z"
+ }
}
'''
\ No newline at end of file
|
