diff --git a/CreateReport.py b/CreateReport.py index 3800b23..fa4d26c 100644 --- a/CreateReport.py +++ b/CreateReport.py @@ -1,4 +1,4 @@ -#!/bin/env python +#!/usr/bin/env python3 # -*- coding: utf-8 -*- import sys import os diff --git a/DeepExploit.py b/DeepExploit.py index 9208cbb..8e65126 100755 --- a/DeepExploit.py +++ b/DeepExploit.py @@ -1,4 +1,4 @@ -#!/bin/env python +#!/usr/bin/env python3 import copy from datetime import datetime import sys @@ -20,9 +20,9 @@ import tensorflow as tf from bs4 import BeautifulSoup from docopt import docopt -from keras.models import * -from keras.layers import * -from keras import backend as K +from tensorflow.keras.models import * +from tensorflow.keras.layers import * +from tensorflow.keras import backend as K from util import Utilty from modules.VersionChecker import VersionChecker from modules.VersionCheckerML import VersionCheckerML @@ -182,10 +182,10 @@ def send_command(self, console_id, command, visualization, sleep=0.1): ret = self.call('console.read', [console_id]) time.sleep(sleep) result = '' - try: - result = ret.get(b'data').decode('utf-8') - if visualization: - self.util.print_message(OK, 'Result of "{}":\n{}'.format(command, result)) + try: + result = self.util.to_str(ret.get(b'data')) + if visualization: + self.util.print_message(OK, 'Result of "{}":\n{}'.format(command, result)) except Exception as e: self.util.print_exception(e, 'Failed: {}'.format(command)) return result @@ -207,10 +207,10 @@ def get_module_list(self, module_type): ret = self.call('module.nops', []) try: - byte_list = ret[b'modules'] - string_list = [] - for module in byte_list: - string_list.append(module.decode('utf-8')) + byte_list = ret[b'modules'] + string_list = [] + for module in byte_list: + string_list.append(self.util.to_str(module)) return string_list except Exception as e: self.util.print_exception(e, 'Failed: Getting {} module list.'.format(module_type)) @@ -224,10 +224,10 @@ def get_module_info(self, module_type, module_name): def get_compatible_payload_list(self, module_name): ret = self.call('module.compatible_payloads', [module_name]) try: - byte_list = ret[b'payloads'] - string_list = [] - for module in byte_list: - string_list.append(module.decode('utf-8')) + byte_list = ret[b'payloads'] + string_list = [] + for module in byte_list: + string_list.append(self.util.to_str(module)) return string_list except Exception as e: self.util.print_exception(e, 'Failed: module.compatible_payloads.') @@ -237,10 +237,10 @@ def get_compatible_payload_list(self, module_name): def get_target_compatible_payload_list(self, module_name, target_num): ret = self.call('module.target_compatible_payloads', [module_name, target_num]) try: - byte_list = ret[b'payloads'] - string_list = [] - for module in byte_list: - string_list.append(module.decode('utf-8')) + byte_list = ret[b'payloads'] + string_list = [] + for module in byte_list: + string_list.append(self.util.to_str(module)) return string_list except Exception as e: self.util.print_exception(e, 'Failed: module.target_compatible_payloads.') @@ -254,9 +254,9 @@ def get_module_options(self, module_type, module_name): def execute_module(self, module_type, module_name, options): ret = self.call('module.execute', [module_type, module_name, options]) try: - job_id = ret[b'job_id'] - uuid = ret[b'uuid'].decode('utf-8') - return job_id, uuid + job_id = ret[b'job_id'] + uuid = self.util.to_str(ret[b'uuid']) + return job_id, uuid except Exception as e: if ret[b'error_code'] == 401: self.login(self.msgrpc_user, self.msgrpc_pass) @@ -268,10 +268,10 @@ def execute_module(self, module_type, module_name, options): def get_job_list(self): jobs = self.call('job.list', []) try: - byte_list = jobs.keys() - job_list = [] - for job_id in byte_list: - job_list.append(int(job_id.decode('utf-8'))) + byte_list = jobs.keys() + job_list = [] + for job_id in byte_list: + job_list.append(int(self.util.to_str(job_id))) return job_list except Exception as e: self.util.print_exception(e, 'Failed: job.list.') @@ -301,7 +301,7 @@ def stop_meterpreter_session(self, session_id): def execute_shell(self, session_id, cmd): ret = self.call('session.shell_write', [str(session_id), cmd]) try: - return ret[b'write_count'].decode('utf-8') + return self.util.to_str(ret[b'write_count']) except Exception as e: self.util.print_exception(e, 'Failed: {}'.format(cmd)) return 'Failed' @@ -310,8 +310,8 @@ def execute_shell(self, session_id, cmd): def get_shell_result(self, session_id, read_pointer): ret = self.call('session.shell_read', [str(session_id), read_pointer]) try: - seq = ret[b'seq'].decode('utf-8') - data = ret[b'data'].decode('utf-8') + seq = self.util.to_str(ret[b'seq']) + data = self.util.to_str(ret[b'data']) return seq, data except Exception as e: self.util.print_exception(e, 'Failed: session.shell_read.') @@ -321,7 +321,7 @@ def get_shell_result(self, session_id, read_pointer): def execute_meterpreter(self, session_id, cmd): ret = self.call('session.meterpreter_write', [str(session_id), cmd]) try: - return ret[b'result'].decode('utf-8') + return self.util.to_str(ret[b'result']) except Exception as e: self.util.print_exception(e, 'Failed: {}'.format(cmd)) return 'Failed' @@ -330,7 +330,7 @@ def execute_meterpreter(self, session_id, cmd): def execute_meterpreter_run_single(self, session_id, cmd): ret = self.call('session.meterpreter_run_single', [str(session_id), cmd]) try: - return ret[b'result'].decode('utf-8') + return self.util.to_str(ret[b'result']) except Exception as e: self.util.print_exception(e, 'Failed: {}'.format(cmd)) return 'Failed' @@ -339,7 +339,7 @@ def execute_meterpreter_run_single(self, session_id, cmd): def get_meterpreter_result(self, session_id): ret = self.call('session.meterpreter_read', [str(session_id)]) try: - return ret[b'data'].decode('utf-8') + return self.util.to_str(ret[b'data']) except Exception as e: self.util.print_exception(e, 'Failed: session.meterpreter_read') return None @@ -348,7 +348,7 @@ def get_meterpreter_result(self, session_id): def upgrade_shell_session(self, session_id, lhost, lport): ret = self.call('session.shell_upgrade', [str(session_id), lhost, lport]) try: - return ret[b'result'].decode('utf-8') + return self.util.to_str(ret[b'result']) except Exception as e: self.util.print_exception(e, 'Failed: session.shell_upgrade') return 'Failed' @@ -486,21 +486,21 @@ def get_exploit_tree(self): sub_option = {} sub_key_list = options[key].keys() for sub_key in sub_key_list: - if isinstance(options[key][sub_key], list): - end_option = [] - for end_key in options[key][sub_key]: - end_option.append(end_key.decode('utf-8')) - sub_option[sub_key.decode('utf-8')] = end_option - else: - end_option = {} - if isinstance(options[key][sub_key], bytes): - sub_option[sub_key.decode('utf-8')] = options[key][sub_key].decode('utf-8') - else: - sub_option[sub_key.decode('utf-8')] = options[key][sub_key] + if isinstance(options[key][sub_key], list): + end_option = [] + for end_key in options[key][sub_key]: + end_option.append(self.util.to_str(end_key)) + sub_option[self.util.to_str(sub_key)] = end_option + else: + end_option = {} + if isinstance(options[key][sub_key], bytes): + sub_option[self.util.to_str(sub_key)] = self.util.to_str(options[key][sub_key]) + else: + sub_option[self.util.to_str(sub_key)] = options[key][sub_key] # User specify. - sub_option['user_specify'] = "" - option[key.decode('utf-8')] = sub_option + sub_option['user_specify'] = "" + option[self.util.to_str(key)] = sub_option # Add payloads and targets to exploit tree. temp_target_tree['target_list'] = target_list @@ -941,7 +941,7 @@ def get_port_list(self, nmap_result_file, rhost, ignore_empty_host=False): self.util.print_message(OK, 'Timeout: "{}"'.format(cat_cmd)) break - nmap_result += ret.get(b'data').decode('utf-8') + nmap_result += self.util.to_str(ret.get(b'data')) status = ret.get(b'busy') if status is False: break @@ -1032,7 +1032,7 @@ def get_exploit_list(self): module_info = self.client.get_module_info('exploit', exploit) time.sleep(0.1) try: - rank = module_info[b'rank'].decode('utf-8') + rank = self.util.to_str(module_info[b'rank']) if rank in {'excellent', 'great', 'good'}: all_exploit_list.append(exploit) self.util.print_message(OK, '{}/{} Loaded exploit: {}'.format(str(idx + 1), @@ -1308,14 +1308,14 @@ def execute_exploit(self, action, thread_name, thread_type, target_list, target_ if len(key_list) != 0: # Probably successfully of exploitation (but unsettled). for key in key_list: - exploit_uuid = sessions[key][b'exploit_uuid'].decode('utf-8') + exploit_uuid = self.util.to_str(sessions[key][b'exploit_uuid']) if uuid == exploit_uuid: # Successfully of exploitation. session_id = int(key) - session_type = sessions[key][b'type'].decode('utf-8') + session_type = self.util.to_str(sessions[key][b'type']) session_port = str(sessions[key][b'session_port']) - session_exploit = sessions[key][b'via_exploit'].decode('utf-8') - session_payload = sessions[key][b'via_payload'].decode('utf-8') + session_exploit = self.util.to_str(sessions[key][b'via_exploit']) + session_payload = self.util.to_str(sessions[key][b'via_payload']) module_info = self.client.get_module_info('exploit', session_exploit) # Checking feasibility of post-exploitation. @@ -1340,12 +1340,12 @@ def execute_exploit(self, action, thread_name, thread_type, target_list, target_ message = 'misfire ' # Gather reporting items. - vuln_name = module_info[b'name'].decode('utf-8') - description = module_info[b'description'].decode('utf-8') + vuln_name = self.util.to_str(module_info[b'name']) + description = self.util.to_str(module_info[b'description']) ref_list = module_info[b'references'] reference = '' - for item in ref_list: - reference += '[' + item[0].decode('utf-8') + ']' + '@' + item[1].decode('utf-8') + '@@' + for item in ref_list: + reference += '[' + self.util.to_str(item[0]) + ']' + '@' + self.util.to_str(item[1]) + '@@' # Save reporting item for report. if thread_type == 'learning': diff --git a/README.md b/README.md index af930c0..7897c31 100644 --- a/README.md +++ b/README.md @@ -17,9 +17,9 @@ All of this being said there are a number of reasons why the project needs work. ### Scrapy Custom Modfication If you simply checkout and run the instructions below to attempt to starting using Deep Exploit you will find the website crawler does not work. This related to the following fiile: - /usr/local/lib/python3.6/dist-packages/scrapy/core/downloader/tls.py + /usr/local/lib/python3.x/dist-packages/scrapy/core/downloader/tls.py -As of 29 Sept there is a strange legacy code in this file that relates to SSLv3. Ironically not 3 or 4 days ago (at the time of this writing) the Scrapy team (great product btw) made an update to the most current release of scrapy to finally remove this code. The drawback? Upgrading to the latest version via pip and a git-based install will tell you that this version requires python 3.7 and above. I will only certify this system works with Python 3.6. +As of 29 Sept there is a strange legacy code in this file that relates to SSLv3. Ironically not 3 or 4 days ago (at the time of this writing) the Scrapy team (great product btw) made an update to the most current release of scrapy to finally remove this code. The drawback? Upgrading to the latest version via pip and a git-based install will tell you that this version requires python 3.7 and above. This fork targets Python 3.10. # Development We provide a copy of the original requirements.txt file used in the upstream project within the **doc/** sub-directory. As we stated above, the assumption is that you are working with Ubuntu Linux 18.04 for and testing or research you may do. @@ -36,7 +36,7 @@ The following setup process should make your Ubuntu 18 machine capable of runnin chmod +x msfinstall sudo ./msfinstall msfdb init - sudo apt install -y autoconf git vim htop net-tools wireshark build-essential libssh2-1-dev libssl-dev python3-pip python3.6-venv rust-all + sudo apt install -y autoconf git vim htop net-tools wireshark build-essential libssh2-1-dev libssl-dev python3-pip python3.10-venv rust-all git clone https://github.com/nmap/nmap.git cd nmap/ ./configure @@ -59,8 +59,17 @@ The following setup process should make your Ubuntu 18 machine capable of runnin sudo apt install -y nvidia-cuda-toolkit sudo cp doc/script/msfrpcd.service /lib/systemd/system sudo systemctl daemon-reload - sudo systemctl enable msfrpcd - sudo systemctl start msfrpcd.service + sudo systemctl enable msfrpcd + sudo systemctl start msfrpcd.service + +## macOS +On macOS you can install the dependencies with [Homebrew](https://brew.sh/): + + brew install metasploit nmap proxychains-ng + python3 -m venv virtualenv + source ./virtualenv/bin/activate + pip install -r requirements.txt + msfdb init ## Kali Linux Although the project does not work against Kali Linux I did not want to lose the original research on how to at least make it functional. @@ -266,23 +275,22 @@ Report sample is below. --- -## Operation check environment - * Hardware - * OS: Kali Linux 2018.2 - * CPU: Intel(R) Core(TM) i7-6500U 2.50GHz - * GPU: None - * Memory: 8.0GB - * Software - * Metasploit Framework 4.16.48-dev - * Python 3.6.5rc1 - * beautifulsoup4==4.6.0 - * docopt==0.6.2 - * Jinja2==2.10 - * Keras==2.1.6 - * msgpack-python==0.5.6 - * numpy==1.13.3 - * pandas==0.23.0 - * tensorflow==1.8.0 +## Operation check environment +* Hardware + * OS: macOS 13 + * CPU: Apple Silicon + * GPU: built-in + * Memory: 8.0GB +* Software + * Metasploit Framework 6 + * Python 3.10 + * beautifulsoup4 4.11 + * docopt 0.6.2 + * Jinja2 3.1 + * msgpack 1.0 + * numpy 1.20 + * pandas 1.3 + * tensorflow-macos 2.13 --- diff --git a/Spider.py b/Spider.py index 3c8810d..bc91907 100644 --- a/Spider.py +++ b/Spider.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python3 # coding:utf-8 import os import time @@ -23,9 +23,11 @@ def __init__(self, category=None, *args, **kwargs): self.response_log = getattr(self, 'response_log', None) msgrpc_host = getattr(self, 'msgrpc_host', None) msgrpc_port = int(getattr(self, 'msgrpc_port', None)) - self.client = Msgrpc({'host': msgrpc_host, 'port': msgrpc_port}) - self.client.console_id = getattr(self, 'msgrpc_console_id', None).encode('utf-8') - self.client.token = getattr(self, 'msgrpc_token', None).encode('utf-8') + self.client = Msgrpc({'host': msgrpc_host, 'port': msgrpc_port}) + console = getattr(self, 'msgrpc_console_id', None) + token = getattr(self, 'msgrpc_token', None) + self.client.console_id = console.encode('utf-8') if isinstance(console, str) else console + self.client.token = token.encode('utf-8') if isinstance(token, str) else token self.client.authenticated = True self.custom_settings = { 'CONCURRENT_REQUESTS': self.concurrent, diff --git a/modules/ContentExplorer.py b/modules/ContentExplorer.py index 12c47a2..073836e 100644 --- a/modules/ContentExplorer.py +++ b/modules/ContentExplorer.py @@ -1,4 +1,4 @@ -#!/bin/env python +#!/usr/bin/env python3 # -*- coding: utf-8 -*- from distutils.log import WARN import os diff --git a/modules/NaiveBayes.py b/modules/NaiveBayes.py index b50fb16..8b8bef6 100644 --- a/modules/NaiveBayes.py +++ b/modules/NaiveBayes.py @@ -1,4 +1,4 @@ -#!/bin/env python +#!/usr/bin/env python3 # -*- coding: utf-8 -*- import os import sys diff --git a/modules/VersionChecker.py b/modules/VersionChecker.py index cf06b3c..7ea2b37 100644 --- a/modules/VersionChecker.py +++ b/modules/VersionChecker.py @@ -1,4 +1,4 @@ -#!/bin/env python +#!/usr/bin/env python3 # -*- coding: utf-8 -*- import os import sys diff --git a/modules/VersionCheckerML.py b/modules/VersionCheckerML.py index 43886ff..bab838f 100644 --- a/modules/VersionCheckerML.py +++ b/modules/VersionCheckerML.py @@ -1,4 +1,4 @@ -#!/bin/env python +#!/usr/bin/env python3 # -*- coding: utf-8 -*- import os import sys diff --git a/requirements.txt b/requirements.txt index babdaed..f612b80 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,13 +1,12 @@ -beautifulsoup4==4.6.3 -docopt==0.6.2 -jinja2>=2.11.3 -Keras==2.2.4 -matplotlib==3.0.3 -msgpack-python==0.5.6 -numpy==1.16.3 -pandas==0.24.2 -Scrapy==2.3.0 -tensorflow>=1.8.0 -urllib3==1.24.2 -protobuf==3.6.1 - +beautifulsoup4>=4.11 +docopt>=0.6.2 +jinja2>=3.1 +matplotlib>=3.5 +msgpack>=1.0 +numpy>=1.20 +pandas>=1.3 +Scrapy>=2.6 +tensorflow-macos>=2.13; platform_system=="Darwin" +tensorflow>=2.13; platform_system!="Darwin" +urllib3>=1.26 +protobuf>=3.20 diff --git a/util.py b/util.py index e2532e1..3c3f146 100644 --- a/util.py +++ b/util.py @@ -1,4 +1,4 @@ -#!/bin/env python +#!/usr/bin/env python3 # -*- coding: utf-8 -*- import os import sys @@ -147,14 +147,19 @@ def transform_date_string(self, target_date): return target_date.strftime(self.report_date_format) # Delete control character. - def delete_ctrl_char(self, origin_text): - clean_text = '' - for char in origin_text: - ord_num = ord(char) - # Allow LF,CR,SP and symbol, character and numeric. - if (ord_num == 10 or ord_num == 13) or (32 <= ord_num <= 126): - clean_text += chr(ord_num) - return clean_text + def delete_ctrl_char(self, origin_text): + clean_text = '' + for char in origin_text: + ord_num = ord(char) + # Allow LF,CR,SP and symbol, character and numeric. + if (ord_num == 10 or ord_num == 13) or (32 <= ord_num <= 126): + clean_text += chr(ord_num) + return clean_text + + # Convert bytes or other objects to string. + @staticmethod + def to_str(value): + return value.decode('utf-8') if isinstance(value, bytes) else str(value) # Check web port. def check_web_port(self, target_ip, port_list, client): @@ -210,12 +215,14 @@ def run_spider(self, target_ip, target_web, client): response_log = target_ip + '_' + target_info[0] + '.log' now_time = self.get_current_date('%Y%m%d%H%M%S') result_file = os.path.join(self.output_base_path, now_time + self.output_filename) - option = ' -a target_url=' + target_url + ' -a allow_domain=' + target_ip + \ - ' -a concurrent=' + self.spider_concurrent_reqs + ' -a depth_limit=' + self.spider_depth_limit + \ - ' -a delay=' + self.spider_delay_time + ' -a store_path=' + self.store_path + \ - ' -a response_log=' + response_log + ' -a msgrpc_host=' + client.host + \ - ' -a msgrpc_port=' + str(client.port) + ' -a msgrpc_token=' + client.token.decode('utf-8') + \ - ' -a msgrpc_console_id=' + client.console_id.decode('utf-8') + ' -o ' + result_file + token = self.to_str(client.token) + console = self.to_str(client.console_id) + option = ' -a target_url=' + target_url + ' -a allow_domain=' + target_ip + \ + ' -a concurrent=' + self.spider_concurrent_reqs + ' -a depth_limit=' + self.spider_depth_limit + \ + ' -a delay=' + self.spider_delay_time + ' -a store_path=' + self.store_path + \ + ' -a response_log=' + response_log + ' -a msgrpc_host=' + client.host + \ + ' -a msgrpc_port=' + str(client.port) + ' -a msgrpc_token=' + token + \ + ' -a msgrpc_console_id=' + console + ' -o ' + result_file close_opton = ' -s CLOSESPIDER_TIMEOUT=' + self.spider_time_out + \ ' -s CLOSESPIDER_ITEMCOUNT=' + self.spider_item_count + \ ' -s CLOSESPIDER_PAGECOUNT=' + self.spider_page_count + \