feat: Integrate TruffleHog secret detection patterns

extract_trufflehog_patterns.py

@@ -0,0 +1,137 @@
+#!/usr/bin/env python3
+
+import os
+import re
+import json
+from pathlib import Path
+
+def extract_patterns_from_go_file(filepath):
+    """Extract regex patterns and keywords from a Go detector file."""
+    patterns = {}
+    keywords = []
+    detector_name = Path(filepath).parent.name
+
+    with open(filepath, 'r') as f:
+        content = f.read()
+
+    # Extract patterns using various common pattern names
+    # Pattern 1: keyPat = regexp.MustCompile(`...`)
+    # Pattern 2: tokenPats = map[string]*regexp.Regexp{ ... }
+    # Pattern 3: var xxxPat = regexp.MustCompile(`...`)
+
+    # Simple single pattern extraction
+    single_patterns = re.findall(r'(\w+Pat)\s*=\s*regexp\.MustCompile\(`([^`]+)`\)', content)
+    for name, pattern in single_patterns:
+        patterns[name] = pattern
+
+    # Map-based patterns (like Slack)
+    map_pattern = re.search(r'tokenPats\s*=\s*map\[string\]\*regexp\.Regexp\{([^}]+)\}', content, re.DOTALL)
+    if map_pattern:
+        map_content = map_pattern.group(1)
+        map_entries = re.findall(r'"([^"]+)":\s*regexp\.MustCompile\(`([^`]+)`\)', map_content)
+        for name, pattern in map_entries:
+            patterns[name] = pattern
+
+    # Extract var SecretPat patterns (AWS style)
+    secret_patterns = re.findall(r'var\s+(\w+)\s*=\s*regexp\.MustCompile\(`([^`]+)`\)', content)
+    for name, pattern in secret_patterns:
+        patterns[name] = pattern
+
+    # Extract keywords
+    keyword_match = re.search(r'func\s+\(\w+\s+\w+\)\s+Keywords\(\)\s+\[\]string\s*\{[^}]*return\s+\[\]string\{([^}]+)\}', content, re.DOTALL)
+    if keyword_match:
+        keyword_content = keyword_match.group(1)
+        keywords = re.findall(r'"([^"]+)"', keyword_content)
+
+    return {
+        'detector': detector_name,
+        'patterns': patterns,
+        'keywords': keywords
+    }
+
+def main():
+    base_path = '/Users/rez0/git/trufflehog/pkg/detectors'
+    all_detectors = []
+
+    # Find all detector Go files (excluding test files)
+    for root, dirs, files in os.walk(base_path):
+        # Skip test directories
+        if 'test' in root:
+            continue
+
+        for file in files:
+            if file.endswith('.go') and not file.endswith('_test.go'):
+                filepath = os.path.join(root, file)
+
+                # Only process main detector files, not integration tests
+                if 'integration' in file or file in ['detectors.go', 'falsepositives.go', 
+                                                      'account_filter.go', 'http.go', 
+                                                      'endpoint_customizer.go', 
+                                                      'multi_part_credential_provider.go']:
+                    continue
+
+                try:
+                    detector_info = extract_patterns_from_go_file(filepath)
+                    if detector_info['patterns']:  # Only include if patterns were found
+                        all_detectors.append(detector_info)
+                        print(f"Extracted patterns from {detector_info['detector']}: {len(detector_info['patterns'])} patterns")
+                except Exception as e:
+                    print(f"Error processing {filepath}: {e}")
+
+    # Save to JSON file
+    output_file = '/Users/rez0/git/data-grep/trufflehog_patterns.json'
+    with open(output_file, 'w') as f:
+        json.dump(all_detectors, f, indent=2)
+
+    print(f"\nExtracted patterns from {len(all_detectors)} detectors")
+    print(f"Saved to {output_file}")
+
+    # Create a simplified version with just patterns for easier use
+    simplified_patterns = {}
+    for detector in all_detectors:
+        for pattern_name, pattern in detector['patterns'].items():
+            # Create a descriptive name
+            full_name = f"{detector['detector']}.{pattern_name}"
+            simplified_patterns[full_name] = {
+                'pattern': pattern,
+                'keywords': detector['keywords']
+            }
+
+    simplified_file = '/Users/rez0/git/data-grep/trufflehog_patterns_simplified.json'
+    with open(simplified_file, 'w') as f:
+        json.dump(simplified_patterns, f, indent=2)
+
+    print(f"Saved simplified patterns to {simplified_file}")
+
+    # Create TypeScript constants file for direct integration
+    ts_file = '/Users/rez0/git/data-grep/packages/frontend/src/trufflehog_patterns.ts'
+    with open(ts_file, 'w') as f:
+        f.write("// Auto-generated from TruffleHog detectors\n")
+        f.write("// Generated by extract_trufflehog_patterns.py\n\n")
+        f.write("export interface SecretPattern {\n")
+        f.write("  name: string;\n")
+        f.write("  pattern: string;\n")
+        f.write("  keywords?: string[];\n")
+        f.write("  category: string;\n")
+        f.write("}\n\n")
+        f.write("export const TRUFFLEHOG_PATTERNS: SecretPattern[] = [\n")
+
+        for detector in all_detectors:
+            for pattern_name, pattern in detector['patterns'].items():
+                # Escape the pattern for TypeScript
+                escaped_pattern = pattern.replace('\\', '\\\\').replace('"', '\\"')
+                f.write("  {\n")
+                f.write(f'    name: "{detector["detector"]}.{pattern_name}",\n')
+                f.write(f'    pattern: "{escaped_pattern}",\n')
+                f.write(f'    category: "{detector["detector"]}",\n')
+                if detector['keywords']:
+                    keywords_str = '", "'.join(detector['keywords'])
+                    f.write(f'    keywords: ["{keywords_str}"],\n')
+                f.write("  },\n")
+
+        f.write("];\n")
+
+    print(f"Generated TypeScript file at {ts_file}")
+
+if __name__ == "__main__":
+    main()

packages/frontend/src/components/results/Results.vue

@@ -153,15 +153,23 @@ const stopSearch = async () => {
     <template #content>
       <div class="flex flex-col h-full">
         <div class="flex justify-between items-center mb-4">
-          <span class="text-xl font-semibold">
-            <i class="fas fa-list mr-2"></i>
-            <template v-if="store.results.searchResults">
-              Matches ({{ store.results.uniqueMatchesCount }} matches)
-            </template>
-            <template v-else-if="store.status.isSearching">
-              Searching...
-            </template>
-          </span>
+          <div class="flex flex-col">
+            <span class="text-xl font-semibold">
+              <i class="fas fa-list mr-2"></i>
+              <template v-if="store.results.searchResults">
+                Matches ({{ store.results.uniqueMatchesCount }} matches)
+              </template>
+              <template v-else-if="store.status.isSearching">
+                Searching...
+              </template>
+            </span>
+            <span v-if="store.currentPatternName" class="text-sm text-gray-400 mt-1 ml-7">
+              Pattern: {{ store.currentPatternName }}
+            </span>
+            <span v-else-if="store.pattern" class="text-sm text-gray-400 mt-1 ml-7">
+              Custom Pattern
+            </span>
+          </div>
           <div class="text-sm text-gray-500 flex items-center gap-2">
             <template v-if="store.status.isSearching">
               <div class="shimmer">Searching {{ store.status.progress }}%</div>

packages/frontend/src/components/search/Form.vue

@@ -1,17 +1,19 @@
 <script setup lang="ts">
 import { GuideContainer } from "@/components/guide";
-import { useAIStore, useGrepStore, useGuideStore, usePatternsStore } from "@/stores";
+import { useAIStore, useGrepStore, useGuideStore, usePatternsStore, useBatchSearchStore } from "@/stores";
 import Button from "primevue/button";
 import Divider from "primevue/divider";
 import Options from "./Options.vue";
 import Search from "./Search.vue";
 import { AIDialogContainer } from "./ai-dialog";
 import { PatternsDialogContainer } from "./patterns";
+import BatchSearchDialog from "./batch/BatchSearchDialog.vue";
 
 const grepStore = useGrepStore();
 const aiStore = useAIStore();
 const patternsStore = usePatternsStore();
 const guideStore = useGuideStore();
+const batchSearchStore = useBatchSearchStore();
 </script>
 
 <template>
@@ -30,7 +32,7 @@ const guideStore = useGuideStore();
     <Options />
 
     <div class="flex justify-between mt-2 gap-2">
-      <div class="flex gap-2">
+      <div class="flex gap-2 flex-wrap">
         <Button
           label="Search"
           icon="fas fa-search"
@@ -39,6 +41,14 @@ const guideStore = useGuideStore();
           :loading="grepStore.status.isSearching"
           :disabled="!grepStore.pattern.trim()"
         />
+        <Button
+          label="Search All Secrets"
+          icon="fas fa-key"
+          class="p-button-warning"
+          @click="batchSearchStore.searchAllSecrets(grepStore.options)"
+          :loading="batchSearchStore.status.isSearching"
+          tooltip="Search for all known secret patterns"
+        />
         <Button
           :loading="aiStore.isProcessing"
           label="Ask AI"
@@ -66,5 +76,6 @@ const guideStore = useGuideStore();
     <AIDialogContainer />
     <PatternsDialogContainer />
     <GuideContainer />
+    <BatchSearchDialog />
   </div>
 </template>

packages/frontend/src/components/search/Search.vue

@@ -3,13 +3,22 @@ import { useGrepStore } from "@/stores";
 import InputText from "primevue/inputtext";
 
 const grepStore = useGrepStore();
+
+// Clear pattern name when user types a custom pattern
+function handlePatternInput(event: Event) {
+  const input = event.target as HTMLInputElement;
+  grepStore.pattern = input.value;
+  // Clear the pattern name since user is entering a custom pattern
+  grepStore.currentPatternName = "";
+}
 </script>
 
 <template>
   <div class="mt-2.5">
     <label class="block mb-2.5 font-medium">Search Pattern (regex)</label>
     <InputText
-      v-model="grepStore.pattern"
+      :value="grepStore.pattern"
+      @input="handlePatternInput"
       class="w-full h-12"
       placeholder="Enter a regex pattern to search for"
       v-tooltip.bottom="'Enter a regular expression to search for'"