Improve RPC string handling

CreateReport.py

@@ -1,4 +1,4 @@
-#!/bin/env python
+#!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 import sys
 import os

DeepExploit.py

@@ -1,4 +1,4 @@
-#!/bin/env python
+#!/usr/bin/env python3
 import copy
 from datetime import datetime
 import sys
@@ -20,9 +20,9 @@
 import tensorflow as tf
 from bs4 import BeautifulSoup
 from docopt import docopt
-from keras.models import *
-from keras.layers import *
-from keras import backend as K
+from tensorflow.keras.models import *
+from tensorflow.keras.layers import *
+from tensorflow.keras import backend as K
 from util import Utilty
 from modules.VersionChecker import VersionChecker
 from modules.VersionCheckerML import VersionCheckerML
@@ -182,10 +182,10 @@ def send_command(self, console_id, command, visualization, sleep=0.1):
         ret = self.call('console.read', [console_id])
         time.sleep(sleep)
         result = ''
-        try:
-            result = ret.get(b'data').decode('utf-8')
-            if visualization:
-                self.util.print_message(OK, 'Result of "{}":\n{}'.format(command, result))
+        try:
+            result = self.util.to_str(ret.get(b'data'))
+            if visualization:
+                self.util.print_message(OK, 'Result of "{}":\n{}'.format(command, result))
         except Exception as e:
             self.util.print_exception(e, 'Failed: {}'.format(command))
         return result
@@ -207,10 +207,10 @@ def get_module_list(self, module_type):
             ret = self.call('module.nops', [])
 
         try:
-            byte_list = ret[b'modules']
-            string_list = []
-            for module in byte_list:
-                string_list.append(module.decode('utf-8'))
+            byte_list = ret[b'modules']
+            string_list = []
+            for module in byte_list:
+                string_list.append(self.util.to_str(module))
             return string_list
         except Exception as e:
             self.util.print_exception(e, 'Failed: Getting {} module list.'.format(module_type))
@@ -224,10 +224,10 @@ def get_module_info(self, module_type, module_name):
     def get_compatible_payload_list(self, module_name):
         ret = self.call('module.compatible_payloads', [module_name])
         try:
-            byte_list = ret[b'payloads']
-            string_list = []
-            for module in byte_list:
-                string_list.append(module.decode('utf-8'))
+            byte_list = ret[b'payloads']
+            string_list = []
+            for module in byte_list:
+                string_list.append(self.util.to_str(module))
             return string_list
         except Exception as e:
             self.util.print_exception(e, 'Failed: module.compatible_payloads.')
@@ -237,10 +237,10 @@ def get_compatible_payload_list(self, module_name):
     def get_target_compatible_payload_list(self, module_name, target_num):
         ret = self.call('module.target_compatible_payloads', [module_name, target_num])
         try:
-            byte_list = ret[b'payloads']
-            string_list = []
-            for module in byte_list:
-                string_list.append(module.decode('utf-8'))
+            byte_list = ret[b'payloads']
+            string_list = []
+            for module in byte_list:
+                string_list.append(self.util.to_str(module))
             return string_list
         except Exception as e:
             self.util.print_exception(e, 'Failed: module.target_compatible_payloads.')
@@ -254,9 +254,9 @@ def get_module_options(self, module_type, module_name):
     def execute_module(self, module_type, module_name, options):
         ret = self.call('module.execute', [module_type, module_name, options])
         try:
-            job_id = ret[b'job_id']
-            uuid = ret[b'uuid'].decode('utf-8')
-            return job_id, uuid
+            job_id = ret[b'job_id']
+            uuid = self.util.to_str(ret[b'uuid'])
+            return job_id, uuid
         except Exception as e:
             if ret[b'error_code'] == 401:
                 self.login(self.msgrpc_user, self.msgrpc_pass)
@@ -268,10 +268,10 @@ def execute_module(self, module_type, module_name, options):
     def get_job_list(self):
         jobs = self.call('job.list', [])
         try:
-            byte_list = jobs.keys()
-            job_list = []
-            for job_id in byte_list:
-                job_list.append(int(job_id.decode('utf-8')))
+            byte_list = jobs.keys()
+            job_list = []
+            for job_id in byte_list:
+                job_list.append(int(self.util.to_str(job_id)))
             return job_list
         except Exception as e:
             self.util.print_exception(e, 'Failed: job.list.')
@@ -301,7 +301,7 @@ def stop_meterpreter_session(self, session_id):
     def execute_shell(self, session_id, cmd):
         ret = self.call('session.shell_write', [str(session_id), cmd])
         try:
-            return ret[b'write_count'].decode('utf-8')
+            return self.util.to_str(ret[b'write_count'])
         except Exception as e:
             self.util.print_exception(e, 'Failed: {}'.format(cmd))
             return 'Failed'
@@ -310,8 +310,8 @@ def execute_shell(self, session_id, cmd):
     def get_shell_result(self, session_id, read_pointer):
         ret = self.call('session.shell_read', [str(session_id), read_pointer])
         try:
-            seq = ret[b'seq'].decode('utf-8')
-            data = ret[b'data'].decode('utf-8')
+            seq = self.util.to_str(ret[b'seq'])
+            data = self.util.to_str(ret[b'data'])
             return seq, data
         except Exception as e:
             self.util.print_exception(e, 'Failed: session.shell_read.')
@@ -321,7 +321,7 @@ def get_shell_result(self, session_id, read_pointer):
     def execute_meterpreter(self, session_id, cmd):
         ret = self.call('session.meterpreter_write', [str(session_id), cmd])
         try:
-            return ret[b'result'].decode('utf-8')
+            return self.util.to_str(ret[b'result'])
         except Exception as e:
             self.util.print_exception(e, 'Failed: {}'.format(cmd))
             return 'Failed'
@@ -330,7 +330,7 @@ def execute_meterpreter(self, session_id, cmd):
     def execute_meterpreter_run_single(self, session_id, cmd):
         ret = self.call('session.meterpreter_run_single', [str(session_id), cmd])
         try:
-            return ret[b'result'].decode('utf-8')
+            return self.util.to_str(ret[b'result'])
         except Exception as e:
             self.util.print_exception(e, 'Failed: {}'.format(cmd))
             return 'Failed'
@@ -339,7 +339,7 @@ def execute_meterpreter_run_single(self, session_id, cmd):
     def get_meterpreter_result(self, session_id):
         ret = self.call('session.meterpreter_read', [str(session_id)])
         try:
-            return ret[b'data'].decode('utf-8')
+            return self.util.to_str(ret[b'data'])
         except Exception as e:
             self.util.print_exception(e, 'Failed: session.meterpreter_read')
             return None
@@ -348,7 +348,7 @@ def get_meterpreter_result(self, session_id):
     def upgrade_shell_session(self, session_id, lhost, lport):
         ret = self.call('session.shell_upgrade', [str(session_id), lhost, lport])
         try:
-            return ret[b'result'].decode('utf-8')
+            return self.util.to_str(ret[b'result'])
         except Exception as e:
             self.util.print_exception(e, 'Failed: session.shell_upgrade')
             return 'Failed'
@@ -486,21 +486,21 @@ def get_exploit_tree(self):
                     sub_option = {}
                     sub_key_list = options[key].keys()
                     for sub_key in sub_key_list:
-                        if isinstance(options[key][sub_key], list):
-                            end_option = []
-                            for end_key in options[key][sub_key]:
-                                end_option.append(end_key.decode('utf-8'))
-                            sub_option[sub_key.decode('utf-8')] = end_option
-                        else:
-                            end_option = {}
-                            if isinstance(options[key][sub_key], bytes):
-                                sub_option[sub_key.decode('utf-8')] = options[key][sub_key].decode('utf-8')
-                            else:
-                                sub_option[sub_key.decode('utf-8')] = options[key][sub_key]
+                        if isinstance(options[key][sub_key], list):
+                            end_option = []
+                            for end_key in options[key][sub_key]:
+                                end_option.append(self.util.to_str(end_key))
+                            sub_option[self.util.to_str(sub_key)] = end_option
+                        else:
+                            end_option = {}
+                            if isinstance(options[key][sub_key], bytes):
+                                sub_option[self.util.to_str(sub_key)] = self.util.to_str(options[key][sub_key])
+                            else:
+                                sub_option[self.util.to_str(sub_key)] = options[key][sub_key]
 
                     # User specify.
-                    sub_option['user_specify'] = ""
-                    option[key.decode('utf-8')] = sub_option
+                    sub_option['user_specify'] = ""
+                    option[self.util.to_str(key)] = sub_option
 
                 # Add payloads and targets to exploit tree.
                 temp_target_tree['target_list'] = target_list
@@ -941,7 +941,7 @@ def get_port_list(self, nmap_result_file, rhost, ignore_empty_host=False):
                         self.util.print_message(OK, 'Timeout: "{}"'.format(cat_cmd))
                         break
 
-                    nmap_result += ret.get(b'data').decode('utf-8')
+                    nmap_result += self.util.to_str(ret.get(b'data'))
                     status = ret.get(b'busy')
                     if status is False:
                         break
@@ -1032,7 +1032,7 @@ def get_exploit_list(self):
                 module_info = self.client.get_module_info('exploit', exploit)
                 time.sleep(0.1)
                 try:
-                    rank = module_info[b'rank'].decode('utf-8')
+                    rank = self.util.to_str(module_info[b'rank'])
                     if rank in {'excellent', 'great', 'good'}:
                         all_exploit_list.append(exploit)
                         self.util.print_message(OK, '{}/{} Loaded exploit: {}'.format(str(idx + 1),
@@ -1308,14 +1308,14 @@ def execute_exploit(self, action, thread_name, thread_type, target_list, target_
             if len(key_list) != 0:
                 # Probably successfully of exploitation (but unsettled).
                 for key in key_list:
-                    exploit_uuid = sessions[key][b'exploit_uuid'].decode('utf-8')
+                    exploit_uuid = self.util.to_str(sessions[key][b'exploit_uuid'])
                     if uuid == exploit_uuid:
                         # Successfully of exploitation.
                         session_id = int(key)
-                        session_type = sessions[key][b'type'].decode('utf-8')
+                        session_type = self.util.to_str(sessions[key][b'type'])
                         session_port = str(sessions[key][b'session_port'])
-                        session_exploit = sessions[key][b'via_exploit'].decode('utf-8')
-                        session_payload = sessions[key][b'via_payload'].decode('utf-8')
+                        session_exploit = self.util.to_str(sessions[key][b'via_exploit'])
+                        session_payload = self.util.to_str(sessions[key][b'via_payload'])
                         module_info = self.client.get_module_info('exploit', session_exploit)
 
                         # Checking feasibility of post-exploitation.
@@ -1340,12 +1340,12 @@ def execute_exploit(self, action, thread_name, thread_type, target_list, target_
                             message = 'misfire '
 
                         # Gather reporting items.
-                        vuln_name = module_info[b'name'].decode('utf-8')
-                        description = module_info[b'description'].decode('utf-8')
+                        vuln_name = self.util.to_str(module_info[b'name'])
+                        description = self.util.to_str(module_info[b'description'])
                         ref_list = module_info[b'references']
                         reference = ''
-                        for item in ref_list:
-                            reference += '[' + item[0].decode('utf-8') + ']' + '@' + item[1].decode('utf-8') + '@@'
+                        for item in ref_list:
+                            reference += '[' + self.util.to_str(item[0]) + ']' + '@' + self.util.to_str(item[1]) + '@@'
 
                         # Save reporting item for report.
                         if thread_type == 'learning':

README.md

@@ -17,9 +17,9 @@ All of this being said there are a number of reasons why the project needs work.
 ### Scrapy Custom Modfication
 If you simply checkout and run the instructions below to attempt to starting using Deep Exploit you will find the website crawler does not work. This related to the following fiile:
 
-    /usr/local/lib/python3.6/dist-packages/scrapy/core/downloader/tls.py
+   /usr/local/lib/python3.x/dist-packages/scrapy/core/downloader/tls.py
 
-As of 29 Sept there is a strange legacy code in this file that relates to SSLv3. Ironically not 3 or 4 days ago (at the time of this writing) the Scrapy team (great product btw) made an update to the most current release of scrapy to finally remove this code. The drawback? Upgrading to the latest version via pip and a git-based install will tell you that this version requires python 3.7 and above. I will only certify this system works with Python 3.6.
+As of 29 Sept there is a strange legacy code in this file that relates to SSLv3. Ironically not 3 or 4 days ago (at the time of this writing) the Scrapy team (great product btw) made an update to the most current release of scrapy to finally remove this code. The drawback? Upgrading to the latest version via pip and a git-based install will tell you that this version requires python 3.7 and above. This fork targets Python 3.10.
 
 # Development
 We provide a copy of the original requirements.txt file used in the upstream project within the **doc/** sub-directory. As we stated above, the assumption is that you are working with Ubuntu Linux 18.04 for and testing or research you may do.
@@ -36,7 +36,7 @@ The following setup process should make your Ubuntu 18 machine capable of runnin
     chmod +x msfinstall 
     sudo ./msfinstall
     msfdb init
-    sudo apt install -y autoconf git vim htop net-tools wireshark build-essential libssh2-1-dev libssl-dev python3-pip python3.6-venv rust-all
+    sudo apt install -y autoconf git vim htop net-tools wireshark build-essential libssh2-1-dev libssl-dev python3-pip python3.10-venv rust-all
     git clone https://github.com/nmap/nmap.git
     cd nmap/
     ./configure
@@ -59,8 +59,17 @@ The following setup process should make your Ubuntu 18 machine capable of runnin
     sudo apt install -y nvidia-cuda-toolkit
     sudo cp doc/script/msfrpcd.service /lib/systemd/system
     sudo systemctl daemon-reload
-    sudo systemctl enable msfrpcd
-    sudo systemctl start msfrpcd.service 
+    sudo systemctl enable msfrpcd
+    sudo systemctl start msfrpcd.service
+
+## macOS
+On macOS you can install the dependencies with [Homebrew](https://brew.sh/):
+
+    brew install metasploit nmap proxychains-ng
+    python3 -m venv virtualenv
+    source ./virtualenv/bin/activate
+    pip install -r requirements.txt
+    msfdb init
 
 ## Kali Linux
 Although the project does not work against Kali Linux I did not want to lose the original research on how to at least make it functional. 
@@ -266,23 +275,22 @@ Report sample is below.
 
 ---
 
-## Operation check environment
- * Hardware  
-   * OS: Kali Linux 2018.2  
-   * CPU: Intel(R) Core(TM) i7-6500U 2.50GHz  
-   * GPU: None  
-   * Memory: 8.0GB  
- * Software  
-   * Metasploit Framework 4.16.48-dev  
-   * Python 3.6.5rc1  
-   * beautifulsoup4==4.6.0  
-   * docopt==0.6.2  
-   * Jinja2==2.10  
-   * Keras==2.1.6  
-   * msgpack-python==0.5.6  
-   * numpy==1.13.3  
-   * pandas==0.23.0  
-   * tensorflow==1.8.0  
+## Operation check environment
+* Hardware
+  * OS: macOS 13
+  * CPU: Apple Silicon
+  * GPU: built-in
+  * Memory: 8.0GB
+* Software
+  * Metasploit Framework 6
+  * Python 3.10
+  * beautifulsoup4 4.11
+  * docopt 0.6.2
+  * Jinja2 3.1
+  * msgpack 1.0
+  * numpy 1.20
+  * pandas 1.3
+  * tensorflow-macos 2.13
 
 ---
 

Spider.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env python3
 # coding:utf-8
 import os
 import time
@@ -23,9 +23,11 @@ def __init__(self, category=None, *args, **kwargs):
         self.response_log = getattr(self, 'response_log', None)
         msgrpc_host = getattr(self, 'msgrpc_host', None)
         msgrpc_port = int(getattr(self, 'msgrpc_port', None))
-        self.client = Msgrpc({'host': msgrpc_host, 'port': msgrpc_port})
-        self.client.console_id = getattr(self, 'msgrpc_console_id', None).encode('utf-8')
-        self.client.token = getattr(self, 'msgrpc_token', None).encode('utf-8')
+        self.client = Msgrpc({'host': msgrpc_host, 'port': msgrpc_port})
+        console = getattr(self, 'msgrpc_console_id', None)
+        token = getattr(self, 'msgrpc_token', None)
+        self.client.console_id = console.encode('utf-8') if isinstance(console, str) else console
+        self.client.token = token.encode('utf-8') if isinstance(token, str) else token
         self.client.authenticated = True
         self.custom_settings = {
             'CONCURRENT_REQUESTS': self.concurrent,

modules/ContentExplorer.py

@@ -1,4 +1,4 @@
-#!/bin/env python
+#!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 from distutils.log import WARN
 import os

modules/NaiveBayes.py

@@ -1,4 +1,4 @@
-#!/bin/env python
+#!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 import os
 import sys

modules/VersionChecker.py

@@ -1,4 +1,4 @@
-#!/bin/env python
+#!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 import os
 import sys

modules/VersionCheckerML.py

@@ -1,4 +1,4 @@
-#!/bin/env python
+#!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 import os
 import sys