From 167829704adc3eec58ce6ab3084910783cb7442b Mon Sep 17 00:00:00 2001 From: Charlotte Croce Date: Fri, 18 Apr 2025 14:51:43 -0400 Subject: [PATCH] update search command to use FTS5 SQLite table for complex searches --- src/handlers/config/config_handler.js | 88 --- src/handlers/fylgja_command_handler.js | 291 ++++++---- src/handlers/sigma/sigma_search_handler.js | 168 +++++- src/lang/command_patterns.js | 17 +- src/lang/query_parser.js | 197 +++++++ src/services/sigma/sigma_search_service.js | 111 +++- src/sigma_db/sigma_db_initialize.js | 130 ++++- src/sigma_db/sigma_db_queries.js | 624 ++++++++++++++++++++- 8 files changed, 1359 insertions(+), 267 deletions(-) create mode 100644 src/lang/query_parser.js diff --git a/src/handlers/config/config_handler.js b/src/handlers/config/config_handler.js index ff58672..e69de29 100644 --- a/src/handlers/config/config_handler.js +++ b/src/handlers/config/config_handler.js @@ -1,88 +0,0 @@ -// -// config_handler.js -// handle the /sigma-config command -// -const util = require('util'); -const { exec } = require('child_process'); -const { SIGMA_CLI_PATH } = require('../../config/constants'); -const { loadConfig, updateConfig } = require('../../config/config-manager'); -const { updateSigmaDatabase } = require('../../services/sigma/sigma_repository_service'); -const logger = require('../../utils/logger'); - -// Promisify exec for async/await usage -const execPromise = util.promisify(exec); - -module.exports = (app) => { - app.command('/sigma-config', async ({ command, ack, respond }) => { - await ack(); - logger.info(`Sigma config command received: ${command.text}`); - - const args = command.text.split(' '); - - if (args.length === 0 || args[0] === '') { - // Display current configuration - const config = loadConfig(); - logger.info('Displaying current configuration'); - await respond(`Current configuration:\nSIEM: ${config.siem}\nLanguage: ${config.lang}\nOutput: ${config.output}`); - return; - } - - const configType = args[0]; - - if (configType === 'update') { - logger.info('Starting database update from command'); - try { - await respond('Updating Sigma database... This may take a moment.'); - await updateSigmaDatabase(); - logger.info('Database update completed from command'); - await respond('Sigma database updated successfully'); - } catch (error) { - logger.error(`Database update failed: ${error.message}`); - await respond(`Error updating Sigma database: ${error.message}`); - } - return; - } - - if (args.length < 2) { - logger.warn(`Invalid config command format: ${command.text}`); - await respond(`Invalid command format. Usage: /sigma-config ${configType} [value]`); - return; - } - - const configValue = args[1]; - const config = loadConfig(); - - if (configType === 'siem') { - // Verify the SIEM backend is installed - logger.info(`Attempting to change SIEM to: ${configValue}`); - try { - await execPromise(`${SIGMA_CLI_PATH} list targets | grep ${configValue}`); - updateConfig('siem', configValue); - logger.info(`SIEM configuration updated to: ${configValue}`); - await respond(`SIEM configuration updated to: ${configValue}`); - } catch (error) { - logger.error(`SIEM backend '${configValue}' not found or not installed`); - await respond(`Error: SIEM backend '${configValue}' not found or not installed. Please install it with: sigma plugin install ${configValue}`); - } - } else if (configType === 'lang') { - logger.info(`Changing language to: ${configValue}`); - updateConfig('lang', configValue); - await respond(`Language configuration updated to: ${configValue}`); - } else if (configType === 'output') { - // Check if output format is supported by the current backend - logger.info(`Attempting to change output format to: ${configValue}`); - try { - await execPromise(`${SIGMA_CLI_PATH} list formats ${config.siem} | grep ${configValue}`); - updateConfig('output', configValue); - logger.info(`Output configuration updated to: ${configValue}`); - await respond(`Output configuration updated to: ${configValue}`); - } catch (error) { - logger.error(`Output format '${configValue}' not supported by SIEM backend '${config.siem}'`); - await respond(`Error: Output format '${configValue}' not supported by SIEM backend '${config.siem}'. Run 'sigma list formats ${config.siem}' to see available formats.`); - } - } else { - logger.warn(`Unknown configuration type: ${configType}`); - await respond(`Unknown configuration type: ${configType}. Available types: siem, lang, output, update`); - } - }); -}; \ No newline at end of file diff --git a/src/handlers/fylgja_command_handler.js b/src/handlers/fylgja_command_handler.js index 191e0b2..f9b2b37 100644 --- a/src/handlers/fylgja_command_handler.js +++ b/src/handlers/fylgja_command_handler.js @@ -1,142 +1,209 @@ /** * fylgja_command_handler.js * - * Unified command handler for the Fylgja Slack bot. - * Processes natural language commands and routes to appropriate handlers. + * Main handler for the /fylgja slash command + * Parses natural language commands and routes to appropriate handlers */ + const logger = require('../utils/logger'); +const { parseCommand } = require('../lang/command_parser'); const { handleError } = require('../utils/error_handler'); +const { handleCommand: handleSigmaSearch, handleComplexSearch } = require('./sigma/sigma_search_handler'); +const { handleCommand: handleSigmaDetails } = require('./sigma/sigma_details_handler'); +const { handleCommand: handleSigmaStats } = require('./sigma/sigma_stats_handler'); +const { handleCommand: handleSigmaCreate } = require('./sigma/sigma_create_handler'); +const { handleCommand: handleAlerts } = require('./alerts/alerts_handler'); +const { handleCommand: handleCase } = require('./case/case_handler'); +const { handleCommand: handleConfig } = require('./config/config_handler'); +const { handleCommand: handleStats } = require('./stats/stats_handler'); + const FILE_NAME = 'fylgja_command_handler.js'; -// Import command handlers -const sigmaDetailsHandler = require('./sigma/sigma_details_handler'); -const sigmaSearchHandler = require('./sigma/sigma_search_handler'); -const sigmaCreateHandler = require('./sigma/sigma_create_handler'); -const sigmaStatsHandler = require('./sigma/sigma_stats_handler'); - -// Import language processing utilities -const commandParser = require('../lang/command_parser'); - /** - * Handle the universal fylgja command + * Main handler for the /fylgja command + * Parses natural language input and routes to appropriate module handlers * * @param {Object} command - The Slack command object * @param {Function} respond - Function to send response back to Slack */ const handleCommand = async (command, respond) => { - try { - if (!command || !command.text) { - logger.warn(`${FILE_NAME}: Empty command received for fylgja`); - await respond({ - text: 'Please provide a command. Try `/fylgja help` for available commands.', - response_type: 'ephemeral' - }); - return; - } + try { + logger.info(`${FILE_NAME}: Received command: ${command.text}`); - logger.info(`${FILE_NAME}: Processing fylgja command: ${command.text}`); - - // Parse the natural language command - const parsedCommand = await commandParser.parseCommand(command.text); - - if (!parsedCommand.success) { - logger.warn(`${FILE_NAME}: Failed to parse command: ${command.text}`); - await respond({ - text: `I couldn't understand that command. ${parsedCommand.message || ''}`, - response_type: 'ephemeral' - }); - return; + if (!command.text.trim()) { + logger.warn(`${FILE_NAME}: Empty command received`); + await respond({ + text: "Please provide a command. Try `/fylgja help` for usage examples.", + response_type: 'ephemeral' + }); + return; + } + + // Parse the natural language command + const parsedCommand = await parseCommand(command.text); + logger.debug(`${FILE_NAME}: Parsed command result: ${JSON.stringify(parsedCommand)}`); + + if (!parsedCommand.success) { + logger.warn(`${FILE_NAME}: Command parsing failed: ${parsedCommand.message}`); + await respond({ + text: parsedCommand.message || "I couldn't understand that command. Try `/fylgja help` for examples.", + response_type: 'ephemeral' + }); + return; + } + + // Extract the structured command + const { action, module, params } = parsedCommand.command; + logger.info(`${FILE_NAME}: Routing command - Module: ${module}, Action: ${action}`); + + // Route to the appropriate handler based on module and action + switch (module) { + case 'sigma': + await handleSigmaCommand(action, params, command, respond); + break; + + case 'alerts': + await handleAlerts(command, respond); + break; + + case 'case': + await handleCase(command, respond); + break; + + case 'config': + await handleConfig(command, respond); + break; + + case 'stats': + await handleStats(command, respond); + break; + + case 'help': + await handleHelpCommand(respond); + break; + + default: + logger.warn(`${FILE_NAME}: Unknown module: ${module}`); + await respond({ + text: `Unknown command module: ${module}. Try \`/fylgja help\` for usage examples.`, + response_type: 'ephemeral' + }); + } + } catch (error) { + await handleError(error, `${FILE_NAME}: Command handler`, respond, { + responseType: 'ephemeral' + }); } - - // Route to the appropriate handler based on the parsed command - await routeCommand(parsedCommand.command, command, respond); - - } catch (error) { - await handleError(error, `${FILE_NAME}: Command handler`, respond, { - responseType: 'ephemeral' - }); - } }; /** - * Route the command to the appropriate handler + * Handle Sigma-related commands * - * @param {Object} parsedCommand - The parsed command object - * @param {Object} originalCommand - The original Slack command object - * @param {Function} respond - Function to send response back to Slack + * @param {string} action - The action to perform + * @param {Array} params - Command parameters + * @param {Object} command - The original Slack command + * @param {Function} respond - Function to send response */ -const routeCommand = async (parsedCommand, originalCommand, respond) => { - const { action, module, params } = parsedCommand; - - // Create a modified command object with the extracted parameters - const modifiedCommand = { - ...originalCommand, - text: params.join(' ') - }; - - // Log the routing decision - logger.debug(`${FILE_NAME}: Routing command - Action: ${action}, Module: ${module}, Params: ${JSON.stringify(params)}`); - - // Route to the appropriate handler - switch (`${module}:${action}`) { - case 'sigma:details': - case 'sigma:explain': - await sigmaDetailsHandler.handleCommand(modifiedCommand, respond); - break; - - case 'sigma:search': - await sigmaSearchHandler.handleCommand(modifiedCommand, respond); - break; - - case 'sigma:create': - await sigmaCreateHandler.handleCommand(modifiedCommand, respond); - break; - - case 'sigma:stats': - await sigmaStatsHandler.handleCommand(modifiedCommand, respond); - break; - - case 'help:general': - await showHelp(respond); - break; - - default: - logger.warn(`${FILE_NAME}: Unknown command combination: ${module}:${action}`); - await respond({ - text: `I don't know how to ${action} in ${module}. Try \`/fylgja help\` for available commands.`, - response_type: 'ephemeral' - }); - } +const handleSigmaCommand = async (action, params, command, respond) => { + logger.debug(`${FILE_NAME}: Handling Sigma command - Action: ${action}, Params: ${JSON.stringify(params)}`); + + try { + switch (action) { + case 'search': + // Update the command object with the keyword parameter + command.text = params[0] || ''; + await handleSigmaSearch(command, respond); + break; + + case 'complexSearch': + // Update the command object with the complex query + command.text = params[0] || ''; + await handleComplexSearch(command, respond); + break; + + case 'details': + // Update the command object with the rule ID parameter + command.text = params[0] || ''; + await handleSigmaDetails(command, respond); + break; + + case 'stats': + await handleSigmaStats(command, respond); + break; + + case 'create': + // Update the command object with the rule ID parameter + command.text = params[0] || ''; + await handleSigmaCreate(command, respond); + break; + + default: + logger.warn(`${FILE_NAME}: Unknown Sigma action: ${action}`); + await respond({ + text: `Unknown Sigma action: ${action}. Try \`/fylgja help\` for usage examples.`, + response_type: 'ephemeral' + }); + } + } catch (error) { + await handleError(error, `${FILE_NAME}: Sigma command handler`, respond, { + responseType: 'ephemeral' + }); + } }; /** - * Show help information + * Handle help command * - * @param {Function} respond - Function to send response back to Slack + * @param {Function} respond - Function to send response */ -const showHelp = async (respond) => { - await respond({ - text: "Here are some example commands you can use with Fylgja:", - blocks: [ - { - type: "section", - text: { - type: "mrkdwn", - text: "*Fylgja Commands*\nHere are some example commands you can use:" - } - }, - { - type: "section", - text: { - type: "mrkdwn", - text: "• `/fylgja explain rule from sigma where id=`\n• `/fylgja search sigma for `\n• `/fylgja create rule in sigma with `\n• `/fylgja show stats for sigma`" - } - } - ], - response_type: 'ephemeral' - }); +const handleHelpCommand = async (respond) => { + try { + const helpText = ` +*Fylgja Command Help* + +*Basic Commands:* +• \`/fylgja search \` - Search for Sigma rules by keyword +• \`/fylgja details \` - Get details about a specific Sigma rule +• \`/fylgja stats\` - Get statistics about Sigma rules database + +*Advanced Search Commands:* +• \`/fylgja search sigma rules where title contains "ransomware"\` - Search by title +• \`/fylgja find rules where tags include privilege_escalation\` - Search by tags +• \`/fylgja search rules where logsource.category == "process_creation"\` - Search by log source +• \`/fylgja find rules where modified after 2024-01-01\` - Search by modification date +• \`/fylgja search where level is "high" and tags include "attack.t1055"\` - Combined search + +*Supported Conditions:* +• Title: \`title contains "text"\` +• Description: \`description contains "text"\` +• Log Source: \`logsource.category == "value"\`, \`logsource.product == "value"\` +• Tags: \`tags include "value"\` +• Dates: \`modified after YYYY-MM-DD\`, \`modified before YYYY-MM-DD\` +• Author: \`author is "name"\` +• Level: \`level is "high"\` + +*Logical Operators:* +• AND: \`condition1 AND condition2\` +• OR: \`condition1 OR condition2\` + +*Pagination:* +• Add \`page=N\` to see page N of results +• Add \`limit=N\` to change number of results per page + +For more information, visit the Fylgja documentation. + `; + + await respond({ + text: helpText, + response_type: 'ephemeral' + }); + } catch (error) { + await handleError(error, `${FILE_NAME}: Help command handler`, respond, { + responseType: 'ephemeral' + }); + } }; module.exports = { - handleCommand + handleCommand }; \ No newline at end of file diff --git a/src/handlers/sigma/sigma_search_handler.js b/src/handlers/sigma/sigma_search_handler.js index e161b7d..f6d9e17 100644 --- a/src/handlers/sigma/sigma_search_handler.js +++ b/src/handlers/sigma/sigma_search_handler.js @@ -3,11 +3,11 @@ * * Handles Sigma rule search requests from Slack commands */ -const { searchSigmaRules } = require('../../services/sigma/sigma_search_service'); + +const { searchSigmaRules, searchSigmaRulesComplex } = require('../../services/sigma/sigma_search_service'); const logger = require('../../utils/logger'); const { handleError } = require('../../utils/error_handler'); const { getSearchResultBlocks } = require('../../blocks/sigma/sigma_search_results_block'); - const { getFileName } = require('../../utils/file_utils'); const FILE_NAME = getFileName(__filename); @@ -24,18 +24,18 @@ const MAX_RESULTS_THRESHOLD = 99; const handleCommand = async (command, respond) => { try { logger.debug(`${FILE_NAME}: Processing sigma-search command: ${JSON.stringify(command.text)}`); - + if (!command || !command.text) { logger.warn(`${FILE_NAME}: Empty command received for sigma-search`); await respond('Invalid command. Usage: /sigma-search [keyword]'); return; } - + // Extract search keyword and check for pagination parameters let keyword = command.text.trim(); let page = 1; let pageSize = MAX_RESULTS_PER_PAGE; - + // Check for pagination format: keyword page=X const pagingMatch = keyword.match(/(.+)\s+page=(\d+)$/i); if (pagingMatch) { @@ -43,7 +43,7 @@ const handleCommand = async (command, respond) => { page = parseInt(pagingMatch[2], 10) || 1; logger.debug(`${FILE_NAME}: Detected pagination request: "${keyword}" page ${page}`); } - + // Check for page size format: keyword limit=X const limitMatch = keyword.match(/(.+)\s+limit=(\d+)$/i); if (limitMatch) { @@ -53,29 +53,27 @@ const handleCommand = async (command, respond) => { pageSize = Math.min(Math.max(pageSize, 1), 100); logger.debug(`${FILE_NAME}: Detected page size request: "${keyword}" limit ${pageSize}`); } - + if (!keyword) { logger.warn(`${FILE_NAME}: Missing keyword in sigma-search command`); await respond('Invalid command: missing keyword. Usage: /sigma-search [keyword]'); return; } - + logger.info(`${FILE_NAME}: Searching for rules with keyword: ${keyword} (page ${page}, size ${pageSize})`); logger.debug(`${FILE_NAME}: Search keyword length: ${keyword.length}`); - + await respond({ text: 'Searching for rules... This may take a moment.', response_type: 'ephemeral' }); - + // Search for rules using the service function with pagination const searchResult = await searchSigmaRules(keyword, page, pageSize); - logger.debug(`${FILE_NAME}: Search result status: ${searchResult.success}`); logger.debug(`${FILE_NAME}: Found ${searchResult.results?.length || 0} results out of ${searchResult.pagination?.totalResults || 0} total matches`); - logger.debug(`${FILE_NAME}: About to generate blocks for search results`); - + if (!searchResult.success) { logger.error(`${FILE_NAME}: Search failed: ${searchResult.message}`); await respond({ @@ -84,18 +82,17 @@ const handleCommand = async (command, respond) => { }); return; } - + // Get total count for validation const totalCount = searchResult.pagination?.totalResults || 0; - + // Check if search returned too many results if (totalCount > MAX_RESULTS_THRESHOLD) { logger.warn(`${FILE_NAME}: Search for "${keyword}" returned too many results (${totalCount}), displaying first page with warning`); - // Continue processing but add a notification searchResult.tooManyResults = true; } - + if (!searchResult.results || searchResult.results.length === 0) { if (totalCount > 0) { logger.warn(`${FILE_NAME}: No rules found on page ${page} for "${keyword}", but ${totalCount} total matches exist`); @@ -112,16 +109,14 @@ const handleCommand = async (command, respond) => { } return; } - + // Generate blocks with pagination support let blocks; try { logger.debug(`${FILE_NAME}: Calling getSearchResultBlocks with ${searchResult.results.length} results`); - // If we have too many results, add a warning block at the beginning if (searchResult.tooManyResults) { blocks = getSearchResultBlocks(keyword, searchResult.results, searchResult.pagination); - // Insert warning at the beginning of blocks (after the header) blocks.splice(1, 0, { "type": "section", @@ -133,7 +128,6 @@ const handleCommand = async (command, respond) => { } else { blocks = getSearchResultBlocks(keyword, searchResult.results, searchResult.pagination); } - logger.debug(`${FILE_NAME}: Successfully generated ${blocks?.length || 0} blocks`); } catch (blockError) { // Use error handler for block generation errors @@ -143,19 +137,19 @@ const handleCommand = async (command, respond) => { }); return; } - + // Add debug log before sending response logger.debug(`${FILE_NAME}: About to send response with ${blocks?.length || 0} blocks`); - + // Determine if this should be visible to everyone or just the user const isEphemeral = totalCount > 20; - + // Respond with the search results await respond({ blocks: blocks, response_type: isEphemeral ? 'ephemeral' : 'in_channel' }); - + // Add debug log after sending response logger.debug(`${FILE_NAME}: Response sent successfully`); } catch (error) { @@ -166,6 +160,128 @@ const handleCommand = async (command, respond) => { } }; +/** + * Handle the complex search command for Sigma rules + * Processes advanced search queries with multiple conditions + * + * @param {Object} command - The Slack command object + * @param {Function} respond - Function to send response back to Slack + */ +const handleComplexSearch = async (command, respond) => { + try { + logger.debug(`${FILE_NAME}: Processing complex search command: ${JSON.stringify(command.text)}`); + + if (!command || !command.text) { + logger.warn(`${FILE_NAME}: Empty command received for complex search`); + await respond('Invalid command. Usage: /sigma-search where [conditions]'); + return; + } + + // Extract query string + let queryString = command.text.trim(); + let page = 1; + let pageSize = MAX_RESULTS_PER_PAGE; + + // Check for pagination format: query page=X + const pagingMatch = queryString.match(/(.+)\s+page=(\d+)$/i); + if (pagingMatch) { + queryString = pagingMatch[1].trim(); + page = parseInt(pagingMatch[2], 10) || 1; + logger.debug(`${FILE_NAME}: Detected pagination request in complex search: page ${page}`); + } + + // Check for page size format: query limit=X + const limitMatch = queryString.match(/(.+)\s+limit=(\d+)$/i); + if (limitMatch) { + queryString = limitMatch[1].trim(); + pageSize = parseInt(limitMatch[2], 10) || MAX_RESULTS_PER_PAGE; + // Ensure the page size is within reasonable limits + pageSize = Math.min(Math.max(pageSize, 1), 100); + logger.debug(`${FILE_NAME}: Detected page size request in complex search: limit ${pageSize}`); + } + + logger.info(`${FILE_NAME}: Performing complex search with query: ${queryString}`); + + await respond({ + text: 'Processing complex search query... This may take a moment.', + response_type: 'ephemeral' + }); + + // Perform the complex search + const searchResult = await searchSigmaRulesComplex(queryString, page, pageSize); + + if (!searchResult.success) { + logger.error(`${FILE_NAME}: Complex search failed: ${searchResult.message}`); + await respond({ + text: `Search failed: ${searchResult.message}`, + response_type: 'ephemeral' + }); + return; + } + + // Check if we have results + if (!searchResult.results || searchResult.results.length === 0) { + logger.warn(`${FILE_NAME}: No rules found matching complex query criteria`); + await respond({ + text: `No rules found matching the specified criteria.`, + response_type: 'ephemeral' + }); + return; + } + + // Generate blocks with pagination support + let blocks; + try { + // Use the standard search result blocks but with a modified header + blocks = getSearchResultBlocks( + `Complex Query: ${queryString}`, + searchResult.results, + searchResult.pagination + ); + + // Replace the header to indicate it's a complex search + if (blocks && blocks.length > 0) { + blocks[0] = { + type: "header", + text: { + type: "plain_text", + text: `Sigma Rule Search Results - Query`, + emoji: true + } + }; + + // Add a description of the search criteria + blocks.splice(1, 0, { + type: "section", + text: { + type: "mrkdwn", + text: `*Query:* \`${queryString}\`` + } + }); + } + } catch (blockError) { + await handleError(blockError, `${FILE_NAME}: Complex search block generation`, respond, { + responseType: 'ephemeral', + customMessage: `Error generating results view: ${blockError.message}` + }); + return; + } + + // Respond with the search results + await respond({ + blocks: blocks, + response_type: 'ephemeral' // Complex searches are usually more specific to the user + }); + + logger.info(`${FILE_NAME}: Complex search response sent successfully with ${searchResult.results.length} results`); + } catch (error) { + await handleError(error, `${FILE_NAME}: Complex search handler`, respond, { + responseType: 'ephemeral' + }); + } +}; + module.exports = { - handleCommand + handleCommand, + handleComplexSearch }; \ No newline at end of file diff --git a/src/lang/command_patterns.js b/src/lang/command_patterns.js index 22f3e98..bcd38f3 100644 --- a/src/lang/command_patterns.js +++ b/src/lang/command_patterns.js @@ -30,21 +30,14 @@ const commandPatterns = [ module: 'sigma', params: [2] // rule ID is in capturing group 2 }, - + // Sigma search patterns { name: 'sigma-search', - regex: /^(search|find|look\s+for)\s+(rules|detections)?\s*(in|from)?\s*sigma\s+(for|where|with)?\s+(.+)$/i, - action: 'search', + regex: /^(search|find)\s+(sigma\s+)?(rules|detections)?\s*(where|with)\s+(.+)$/i, + action: 'complexSearch', module: 'sigma', - params: [5] // search query is in capturing group 5 - }, - { - name: 'sigma-search-simple', - regex: /^(search|find)\s+(.+)$/i, - action: 'search', - module: 'sigma', - params: [2] // search query is in capturing group 2 + params: [5] // complex query conditions in capturing group 5 }, // Sigma create patterns @@ -55,7 +48,7 @@ const commandPatterns = [ module: 'sigma', params: [2] // rule ID is in capturing group 2 }, - + // Sigma stats patterns { name: 'sigma-stats-first', diff --git a/src/lang/query_parser.js b/src/lang/query_parser.js new file mode 100644 index 0000000..0f949cc --- /dev/null +++ b/src/lang/query_parser.js @@ -0,0 +1,197 @@ +/** + * query_parser.js + * + * Utility to parse complex search queries for Sigma rules + * Handles conditions like title contains "X", tags include "Y", etc. + */ + +const logger = require('../utils/logger'); +const FILE_NAME = 'query_parser.js'; + +/** + * Parse a complex query string into structured search parameters + * Supports conditions like: + * - title contains "ransomware" + * - logsource.category == "process_creation" + * - tags include privilege_escalation + * - modified after 2024-01-01 + * - author is "John Doe" + * + * Also supports logical operators: + * - AND, and + * - OR, or + * + * @param {string} queryString - The complex query string to parse + * @returns {Object} Structured search parameters + */ +function parseComplexQuery(queryString) { + try { + logger.debug(`${FILE_NAME}: Parsing complex query: ${queryString}`); + + if (!queryString || typeof queryString !== 'string') { + logger.warn(`${FILE_NAME}: Invalid query string`); + return { valid: false, error: 'Invalid query string' }; + } + + // Initialize the result object + const result = { + valid: true, + conditions: [], + operator: 'AND' // Default to AND for multiple conditions + }; + + // Check for explicit logical operators + if (/ AND /i.test(queryString)) { + result.operator = 'AND'; + // Split by AND and parse each part + const parts = queryString.split(/ AND /i); + for (const part of parts) { + const condition = parseCondition(part.trim()); + if (condition) { + result.conditions.push(condition); + } + } + } else if (/ OR /i.test(queryString)) { + result.operator = 'OR'; + // Split by OR and parse each part + const parts = queryString.split(/ OR /i); + for (const part of parts) { + const condition = parseCondition(part.trim()); + if (condition) { + result.conditions.push(condition); + } + } + } else { + // Single condition + const condition = parseCondition(queryString.trim()); + if (condition) { + result.conditions.push(condition); + } + } + + // If no valid conditions found, mark as invalid + if (result.conditions.length === 0) { + result.valid = false; + result.error = 'No valid search conditions found'; + } + + logger.debug(`${FILE_NAME}: Parsed query result: ${JSON.stringify(result)}`); + return result; + } catch (error) { + logger.error(`${FILE_NAME}: Error parsing complex query: ${error.message}`); + return { + valid: false, + error: `Error parsing query: ${error.message}` + }; + } +} + +/** + * Parse a single condition from the query string + * + * @param {string} conditionStr - The condition string to parse + * @returns {Object|null} Parsed condition object or null if invalid + */ +function parseCondition(conditionStr) { + logger.debug(`${FILE_NAME}: Parsing condition: ${conditionStr}`); + + // Define regex patterns for different condition types + const patterns = [ + // title contains "value" + { + regex: /^(title|name)\s+(contains|has|like|includes)\s+"?([^"]+)"?$/i, + handler: (matches) => ({ + field: 'title', + operator: 'contains', + value: matches[3].trim() + }) + }, + // description contains "value" + { + regex: /^(description|desc)\s+(contains|has|like|includes)\s+"?([^"]+)"?$/i, + handler: (matches) => ({ + field: 'description', + operator: 'contains', + value: matches[3].trim() + }) + }, + // logsource.category == "value" or logsource.category = "value" + { + regex: /^logsource\.(\w+)\s*(==|=|equals?)\s*"?([^"]+)"?$/i, + handler: (matches) => ({ + field: 'logsource', + subfield: matches[1].toLowerCase(), + operator: 'equals', + value: matches[3].trim() + }) + }, + // tags include "value" or tag contains "value" + { + regex: /^tags?\s+(includes?|contains|has)\s+"?([^"]+)"?$/i, + handler: (matches) => ({ + field: 'tags', + operator: 'contains', + value: matches[2].trim() + }) + }, + // modified after YYYY-MM-DD + { + regex: /^(modified|updated|created|date)\s+(after|before|on|since)\s+"?(\d{4}-\d{2}-\d{2})"?$/i, + handler: (matches) => ({ + field: 'date', + type: matches[1].toLowerCase(), + operator: matches[2].toLowerCase(), + value: matches[3].trim() + }) + }, + // author is "value" or author = "value" + { + regex: /^(author|creator)\s+(is|equals?|==|=)\s+"?([^"]+)"?$/i, + handler: (matches) => ({ + field: 'author', + operator: 'equals', + value: matches[3].trim() + }) + }, + // level is "value" or level = "value" + { + regex: /^(level|severity)\s+(is|equals?|==|=)\s+"?([^"]+)"?$/i, + handler: (matches) => ({ + field: 'level', + operator: 'equals', + value: matches[3].trim() + }) + }, + // id is "value" or id = "value" + { + regex: /^(id|identifier)\s+(is|equals?|==|=)\s+"?([^"]+)"?$/i, + handler: (matches) => ({ + field: 'id', + operator: 'equals', + value: matches[3].trim() + }) + } + ]; + + // Try each pattern + for (const pattern of patterns) { + const matches = conditionStr.match(pattern.regex); + if (matches) { + return pattern.handler(matches); + } + } + + // If we get here, no patterns matched + logger.warn(`${FILE_NAME}: No pattern matched condition: ${conditionStr}`); + + // Default to simple keyword search if no specific pattern matches + return { + field: 'keyword', + operator: 'contains', + value: conditionStr.trim() + }; +} + +module.exports = { + parseComplexQuery +}; \ No newline at end of file diff --git a/src/services/sigma/sigma_search_service.js b/src/services/sigma/sigma_search_service.js index 81b53dc..4d17ad3 100644 --- a/src/services/sigma/sigma_search_service.js +++ b/src/services/sigma/sigma_search_service.js @@ -1,14 +1,15 @@ /** * sigma_search_service.js * - * This service provides functionality for searching Sigma rules by keywords. + * This service provides functionality for searching Sigma rules by keywords and complex queries. * It processes search results and returns them in a structured format. * Supports pagination for large result sets. */ -const { searchRules } = require('../../sigma_db/sigma_db_queries'); + +const { searchRules, searchRulesComplex } = require('../../sigma_db/sigma_db_queries'); +const { parseComplexQuery } = require('../../lang/query_parser'); const logger = require('../../utils/logger'); const { convertSigmaRule } = require('./sigma_converter_service'); - const { getFileName } = require('../../utils/file_utils'); const FILE_NAME = getFileName(__filename); @@ -153,6 +154,109 @@ async function searchSigmaRules(keyword, page = 1, pageSize = 10) { } } +/** + * Searches for Sigma rules using complex query conditions + * Supports filtering by title, logsource, tags, dates, and more + * + * @param {string} queryString - The complex query string to parse + * @param {number} page - Page number (1-based index, default: 1) + * @param {number} pageSize - Number of results per page (default: 10) + * @returns {Promise} Result object with success flag and processed results + */ +async function searchSigmaRulesComplex(queryString, page = 1, pageSize = 10) { + if (!queryString || typeof queryString !== 'string') { + logger.warn(`${FILE_NAME}: Cannot perform complex search: Missing or invalid query string`); + return { + success: false, + message: 'Missing or invalid complex query' + }; + } + + // Validate pagination parameters + if (typeof page !== 'number' || page < 1) { + logger.warn(`${FILE_NAME}: Invalid page number: ${page}, defaulting to 1`); + page = 1; + } + + if (typeof pageSize !== 'number' || pageSize < 1 || pageSize > 100) { + logger.warn(`${FILE_NAME}: Invalid page size: ${pageSize}, defaulting to 10`); + pageSize = 10; + } + + // Calculate the offset based on page number + const offset = (page - 1) * pageSize; + + logger.info(`${FILE_NAME}: Performing complex search with query: "${queryString}" (page ${page}, size ${pageSize})`); + + try { + // Parse the complex query string + const parsedQuery = parseComplexQuery(queryString); + + if (!parsedQuery.valid) { + logger.warn(`${FILE_NAME}: Invalid complex query: ${parsedQuery.error}`); + return { + success: false, + message: `Invalid query: ${parsedQuery.error}` + }; + } + + // Perform the database search with the parsed query + const searchResult = await searchRulesComplex(parsedQuery, pageSize, offset); + + // Defensive handling of possible return formats + let allResults = []; + let totalCount = 0; + + // Handle search results + if (searchResult) { + if (Array.isArray(searchResult.results)) { + allResults = searchResult.results; + totalCount = searchResult.totalCount || 0; + } + } + + if (allResults.length === 0) { + return { + success: true, + results: [], + message: `No rules found matching the complex query criteria`, + pagination: { + currentPage: page, + pageSize: pageSize, + totalPages: Math.ceil(totalCount / pageSize), + totalResults: totalCount, + hasMore: false + } + }; + } + + // Calculate pagination info + const totalPages = Math.ceil(totalCount / pageSize); + const hasMore = (offset + pageSize) < totalCount; + + return { + success: true, + results: allResults, + count: allResults.length, + query: parsedQuery, + pagination: { + currentPage: page, + pageSize: pageSize, + totalPages: totalPages, + totalResults: totalCount, + hasMore: hasMore + } + }; + } catch (error) { + logger.error(`${FILE_NAME}: Error in complex search: ${error.message}`); + logger.debug(`${FILE_NAME}: Error stack: ${error.stack}`); + return { + success: false, + message: `Error performing complex search: ${error.message}` + }; + } +} + /** * Enhanced search that returns fully converted rule objects with pagination support * This is a more expensive operation than basic search @@ -210,5 +314,6 @@ async function searchAndConvertRules(keyword, page = 1, pageSize = 10) { module.exports = { searchSigmaRules, + searchSigmaRulesComplex, searchAndConvertRules }; \ No newline at end of file diff --git a/src/sigma_db/sigma_db_initialize.js b/src/sigma_db/sigma_db_initialize.js index 934e7d5..c5342d5 100644 --- a/src/sigma_db/sigma_db_initialize.js +++ b/src/sigma_db/sigma_db_initialize.js @@ -58,41 +58,49 @@ async function initializeDatabase(db) { return; } - // Create rules table with basic information - const createRulesTableSql = ` - CREATE TABLE sigma_rules ( - id TEXT PRIMARY KEY, - file_path TEXT, - content TEXT, - date DATETIME DEFAULT CURRENT_TIMESTAMP - ) - `; - - db.run(createRulesTableSql, (err) => { + // Drop FTS table if exists + db.run('DROP TABLE IF EXISTS rule_search', (err) => { if (err) { reject(err); return; } - // Create rule_parameters table for individual parameters - const createParamsTableSql = ` - CREATE TABLE rule_parameters ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - rule_id TEXT, - param_name TEXT, - param_value TEXT, - param_type TEXT, - FOREIGN KEY (rule_id) REFERENCES sigma_rules(id) ON DELETE CASCADE + // Create rules table with basic information + const createRulesTableSql = ` + CREATE TABLE sigma_rules ( + id TEXT PRIMARY KEY, + file_path TEXT, + content TEXT, + date DATETIME DEFAULT CURRENT_TIMESTAMP ) `; - db.run(createParamsTableSql, (err) => { + db.run(createRulesTableSql, (err) => { if (err) { reject(err); - } else { - logger.info(`${FILE_NAME}: Database schema initialized`); - resolve(); + return; } + + // Create rule_parameters table for individual parameters + const createParamsTableSql = ` + CREATE TABLE rule_parameters ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + rule_id TEXT, + param_name TEXT, + param_value TEXT, + param_type TEXT, + FOREIGN KEY (rule_id) REFERENCES sigma_rules(id) ON DELETE CASCADE + ) + `; + + db.run(createParamsTableSql, (err) => { + if (err) { + reject(err); + } else { + logger.info(`${FILE_NAME}: Database schema initialized`); + resolve(); + } + }); }); }); }); @@ -100,6 +108,70 @@ async function initializeDatabase(db) { }); } +// Create FTS5 virtual table for full-text search +async function createFtsTable(db) { + return new Promise((resolve, reject) => { + logger.info(`${FILE_NAME}: Creating FTS5 virtual table for full-text search`); + + // Create the FTS5 virtual table + const createFtsTableSql = ` + CREATE VIRTUAL TABLE IF NOT EXISTS rule_search USING fts5( + rule_id, + title, + description, + logsource, + tags, + author, + level, + content, + tokenize="unicode61" + ); + `; + + db.run(createFtsTableSql, (err) => { + if (err) { + logger.error(`${FILE_NAME}: Failed to create FTS5 table: ${err.message}`); + reject(err); + } else { + logger.info(`${FILE_NAME}: FTS5 virtual table created successfully`); + resolve(); + } + }); + }); +} + +// Populate FTS table with rule data for full-text search +async function populateFtsTable(db) { + return new Promise((resolve, reject) => { + logger.info(`${FILE_NAME}: Populating FTS5 table with rule data`); + + // Insert query that aggregates data from both tables + const populateFtsSql = ` + INSERT INTO rule_search(rule_id, title, description, logsource, tags, author, level, content) + SELECT + r.id, + (SELECT param_value FROM rule_parameters WHERE rule_id = r.id AND param_name = 'title' LIMIT 1), + (SELECT param_value FROM rule_parameters WHERE rule_id = r.id AND param_name = 'description' LIMIT 1), + (SELECT param_value FROM rule_parameters WHERE rule_id = r.id AND param_name = 'logsource' LIMIT 1), + (SELECT param_value FROM rule_parameters WHERE rule_id = r.id AND param_name = 'tags' LIMIT 1), + (SELECT param_value FROM rule_parameters WHERE rule_id = r.id AND param_name = 'author' LIMIT 1), + (SELECT param_value FROM rule_parameters WHERE rule_id = r.id AND param_name = 'level' LIMIT 1), + r.content + FROM sigma_rules r + `; + + db.run(populateFtsSql, (err) => { + if (err) { + logger.error(`${FILE_NAME}: Failed to populate FTS5 table: ${err.message}`); + reject(err); + } else { + logger.info(`${FILE_NAME}: FTS5 table populated successfully`); + resolve(); + } + }); + }); +} + // Determine if a YAML document is a Sigma rule function isSigmaRule(doc) { // Check for essential Sigma rule properties @@ -531,6 +603,12 @@ async function main() { // Create indexes await createIndexes(db); + // Create FTS5 table + await createFtsTable(db); + + // Populate FTS5 table with rule data + await populateFtsTable(db); + // Close database connection db.close((err) => { if (err) { @@ -556,5 +634,7 @@ if (require.main === module) { module.exports = { initializeDatabase, importRules, - createIndexes + createIndexes, + createFtsTable, + populateFtsTable }; \ No newline at end of file diff --git a/src/sigma_db/sigma_db_queries.js b/src/sigma_db/sigma_db_queries.js index 1867c79..8fa53f9 100644 --- a/src/sigma_db/sigma_db_queries.js +++ b/src/sigma_db/sigma_db_queries.js @@ -198,6 +198,17 @@ async function searchRules(keyword, limit = 10, offset = 0) { db = await getDbConnection(); logger.debug(`${FILE_NAME}: Database connection established for search`); + // Use FTS5 for faster searching if available + const ftsAvailable = await checkFtsAvailable(db); + + if (ftsAvailable) { + logger.debug(`${FILE_NAME}: Using FTS5 for keyword search`); + return searchRulesFTS(keyword, limit, offset); + } + + // If FTS5 is not available, use the legacy search method + logger.debug(`${FILE_NAME}: FTS5 not available, using legacy search method`); + // First get the total count of matching rules (for pagination info) const countQuery = ` SELECT COUNT(*) as count @@ -264,6 +275,613 @@ async function searchRules(keyword, limit = 10, offset = 0) { } } +/** + * Check if FTS5 virtual table is available + * + * @param {Object} db - Database connection + * @returns {Promise} Whether FTS5 is available + */ +async function checkFtsAvailable(db) { + try { + const result = await new Promise((resolve, reject) => { + db.get("SELECT name FROM sqlite_master WHERE type='table' AND name='rule_search'", (err, row) => { + if (err) { + logger.error(`${FILE_NAME}: Error checking for FTS5 table: ${err.message}`); + reject(err); + } else { + resolve(row !== undefined); + } + }); + }); + + logger.debug(`${FILE_NAME}: FTS5 table availability check: ${result ? 'Available' : 'Not available'}`); + return result; + } catch (error) { + logger.error(`${FILE_NAME}: Error checking FTS availability: ${error.message}`); + return false; + } +} + +/** + * Search for Sigma rules using FTS5 + * Performs a full-text search and returns matching rules with pagination + * + * @param {string} keyword - The keyword to search for + * @param {number} limit - Maximum number of results to return (default: 10) + * @param {number} offset - Number of results to skip (for pagination, default: 0) + * @returns {Promise} Object with results array and total count + */ +async function searchRulesFTS(keyword, limit = 10, offset = 0) { + if (!keyword) { + logger.warn(`${FILE_NAME}: Empty search keyword provided for FTS search`); + return { results: [], totalCount: 0 }; + } + + // Prepare FTS query - add * for prefix matching if not already present + let ftsQuery = keyword.trim(); + if (!ftsQuery.endsWith('*')) { + ftsQuery = `${ftsQuery}*`; + } + + logger.info(`${FILE_NAME}: Performing FTS search with query: "${ftsQuery}" (limit: ${limit}, offset: ${offset})`); + + let db; + try { + db = await getDbConnection(); + logger.debug(`${FILE_NAME}: Database connection established for FTS search`); + + // First get the total count of matching rules + const countQuery = ` + SELECT COUNT(*) as count + FROM rule_search + WHERE rule_search MATCH ? + `; + + const countResult = await new Promise((resolve, reject) => { + db.get(countQuery, [ftsQuery], (err, row) => { + if (err) { + logger.error(`${FILE_NAME}: FTS count query error: ${err.message}`); + reject(err); + } else { + resolve(row || { count: 0 }); + } + }); + }); + + const totalCount = countResult.count; + logger.debug(`${FILE_NAME}: Total matching rules for FTS query "${ftsQuery}": ${totalCount}`); + + // Now get the actual results with pagination + const searchQuery = ` + SELECT rule_id, title + FROM rule_search + WHERE rule_search MATCH ? + ORDER BY rank + LIMIT ? OFFSET ? + `; + + const results = await new Promise((resolve, reject) => { + db.all(searchQuery, [ftsQuery, limit, offset], (err, rows) => { + if (err) { + logger.error(`${FILE_NAME}: FTS search query error: ${err.message}`); + reject(err); + } else { + logger.debug(`${FILE_NAME}: FTS search query returned ${rows ? rows.length : 0} results`); + resolve(rows || []); + } + }); + }); + + logger.debug(`${FILE_NAME}: FTS search results page for query "${ftsQuery}": ${results.length} matches (page ${Math.floor(offset / limit) + 1})`); + + return { + results: results.map(r => ({ id: r.rule_id, title: r.title || r.rule_id })), + totalCount + }; + } catch (error) { + logger.error(`${FILE_NAME}: Error in FTS search operation: ${error.message}`); + logger.debug(`${FILE_NAME}: FTS search error stack: ${error.stack}`); + return { results: [], totalCount: 0 }; + } finally { + if (db) { + try { + await db.close(); + logger.debug(`${FILE_NAME}: Database connection closed after FTS search operation`); + } catch (closeError) { + logger.error(`${FILE_NAME}: Error closing database connection after FTS search: ${closeError.message}`); + } + } + } +} + +/** + * Search for Sigma rules using complex query conditions + * Supports filtering by multiple attributes like title, logsource, tags, etc. + * + * @param {Object} parsedQuery - The parsed query object containing conditions and operator + * @param {number} limit - Maximum number of results to return + * @param {number} offset - Number of results to skip (for pagination) + * @returns {Promise} Object with results array and total count + */ +async function searchRulesComplex(parsedQuery, limit = 10, offset = 0) { + if (!parsedQuery || !parsedQuery.valid) { + logger.warn(`${FILE_NAME}: Invalid query object provided`); + return { results: [], totalCount: 0 }; + } + + logger.info(`${FILE_NAME}: Performing complex search with ${parsedQuery.conditions.length} conditions (limit: ${limit}, offset: ${offset})`); + + let db; + // Declare this at function scope so it's available in the finally block + let usingFts = false; + + try { + db = await getDbConnection(); + logger.debug(`${FILE_NAME}: Database connection established for complex search`); + + // Check if FTS5 is available + const ftsAvailable = await checkFtsAvailable(db); + + if (ftsAvailable) { + logger.debug(`${FILE_NAME}: Using FTS5 for complex search`); + // Set flag that we're using FTS + usingFts = true; + // Pass db connection to searchRulesComplexFTS and let that function manage it + const results = await searchRulesComplexFTS(parsedQuery, limit, offset, db); + return results; + } + + logger.debug(`${FILE_NAME}: FTS5 not available, using legacy complex search method`); + + // Build the SQL query based on the conditions + const { sqlQuery, sqlCountQuery, params } = buildComplexSqlQuery(parsedQuery, limit, offset); + + logger.debug(`${FILE_NAME}: Executing complex search SQL: ${sqlQuery}`); + logger.debug(`${FILE_NAME}: Query parameters: ${JSON.stringify(params)}`); + + // First get the total count of matching results + const countResult = await new Promise((resolve, reject) => { + db.get(sqlCountQuery, params.slice(0, params.length - 2), (err, row) => { + if (err) { + logger.error(`${FILE_NAME}: Complex search count query error: ${err.message}`); + reject(err); + } else { + resolve(row || { count: 0 }); + } + }); + }); + + const totalCount = countResult.count; + logger.debug(`${FILE_NAME}: Total matching rules for complex query: ${totalCount}`); + + // Now get the actual results with pagination + const results = await new Promise((resolve, reject) => { + db.all(sqlQuery, params, (err, rows) => { + if (err) { + logger.error(`${FILE_NAME}: Complex search query error: ${err.message}`); + reject(err); + } else { + logger.debug(`${FILE_NAME}: Complex search query returned ${rows ? rows.length : 0} results`); + resolve(rows || []); + } + }); + }); + + // Format the results + const formattedResults = results.map(r => ({ + id: r.rule_id, + title: r.title || r.rule_id + })); + + logger.debug(`${FILE_NAME}: Returning ${formattedResults.length} results for complex search`); + + return { + results: formattedResults, + totalCount + }; + } catch (error) { + logger.error(`${FILE_NAME}: Error in complex search operation: ${error.message}`); + logger.debug(`${FILE_NAME}: Complex search error stack: ${error.stack}`); + return { results: [], totalCount: 0 }; + } finally { + // IMPORTANT: Only close the db connection if we're not using FTS + // When using FTS, let searchRulesComplexFTS manage the connection + if (db && !usingFts) { + try { + await new Promise((resolve) => db.close(() => resolve())); + logger.debug(`${FILE_NAME}: Database connection closed after complex search operation`); + } catch (closeError) { + logger.error(`${FILE_NAME}: Error closing database after complex search: ${closeError.message}`); + } + } + } +} + +/** + * Search for Sigma rules using complex query conditions with FTS5 + * Uses the FTS5 virtual table for faster text searching + * + * @param {Object} parsedQuery - The parsed query object + * @param {number} limit - Maximum number of results to return + * @param {number} offset - Number of results to skip (for pagination) + * @param {Object} providedDb - Database connection (optional, will create one if not provided) + * @returns {Promise} Object with results array and total count + */ +async function searchRulesComplexFTS(parsedQuery, limit = 10, offset = 0, providedDb = null) { + if (!parsedQuery || !parsedQuery.valid) { + logger.warn(`${FILE_NAME}: Invalid query object provided for FTS complex search`); + return { results: [], totalCount: 0 }; + } + + logger.info(`${FILE_NAME}: Performing complex FTS search with ${parsedQuery.conditions.length} conditions`); + + let db; + let shouldCloseDb = false; + + try { + // Use provided db connection or create a new one + if (providedDb) { + db = providedDb; + } else { + db = await getDbConnection(); + shouldCloseDb = true; + logger.debug(`${FILE_NAME}: Created new database connection for complex FTS search`); + } + + // Build FTS query from conditions + const { ftsQuery, whereClause, params } = buildComplexFtsQuery(parsedQuery); + + logger.debug(`${FILE_NAME}: FTS query: "${ftsQuery}", additional where: ${whereClause ? whereClause : 'none'}`); + logger.debug(`${FILE_NAME}: Query parameters: ${JSON.stringify(params)}`); + + // Build count query + let countQuery; + let countParams; + + if (whereClause) { + countQuery = ` + SELECT COUNT(*) as count + FROM rule_search + WHERE rule_search MATCH ? + AND ${whereClause} + `; + countParams = [ftsQuery, ...params]; + } else { + countQuery = ` + SELECT COUNT(*) as count + FROM rule_search + WHERE rule_search MATCH ? + `; + countParams = [ftsQuery]; + } + + // Get total count + const countResult = await new Promise((resolve, reject) => { + db.get(countQuery, countParams, (err, row) => { + if (err) { + logger.error(`${FILE_NAME}: Complex FTS count query error: ${err.message}`); + reject(err); + } else { + resolve(row || { count: 0 }); + } + }); + }); + + const totalCount = countResult.count; + logger.debug(`${FILE_NAME}: Total matching rules for complex FTS query: ${totalCount}`); + + // Build results query with pagination + let searchQuery; + let searchParams; + + if (whereClause) { + searchQuery = ` + SELECT rule_id, title + FROM rule_search + WHERE rule_search MATCH ? + AND ${whereClause} + ORDER BY rank + LIMIT ? OFFSET ? + `; + searchParams = [ftsQuery, ...params, limit, offset]; + } else { + searchQuery = ` + SELECT rule_id, title + FROM rule_search + WHERE rule_search MATCH ? + ORDER BY rank + LIMIT ? OFFSET ? + `; + searchParams = [ftsQuery, limit, offset]; + } + + // Get paginated results + const results = await new Promise((resolve, reject) => { + db.all(searchQuery, searchParams, (err, rows) => { + if (err) { + logger.error(`${FILE_NAME}: Complex FTS search query error: ${err.message}`); + reject(err); + } else { + logger.debug(`${FILE_NAME}: Complex FTS search query returned ${rows ? rows.length : 0} results`); + resolve(rows || []); + } + }); + }); + + // Format the results + const formattedResults = results.map(r => ({ + id: r.rule_id, + title: r.title || r.rule_id + })); + + logger.debug(`${FILE_NAME}: Returning ${formattedResults.length} results for complex FTS search`); + + return { + results: formattedResults, + totalCount + }; + } catch (error) { + logger.error(`${FILE_NAME}: Error in complex FTS search operation: ${error.message}`); + logger.debug(`${FILE_NAME}: Complex FTS search error stack: ${error.stack}`); + return { results: [], totalCount: 0 }; + } finally { + // Only close the database if we created it AND we're not in the middle of a transaction + if (db && shouldCloseDb) { + try { + await db.close(); + logger.debug(`${FILE_NAME}: Database connection closed after complex FTS search`); + } catch (closeError) { + logger.error(`${FILE_NAME}: Error closing database after complex FTS search: ${closeError.message}`); + } + } + } +} + +/** + * Build FTS query and WHERE clause from parsed query conditions + * + * @param {Object} parsedQuery - The parsed query object + * @returns {Object} Object with FTS query, additional WHERE clause, and parameters + */ +function buildComplexFtsQuery(parsedQuery) { + const { conditions, operator } = parsedQuery; + + // Separate text search conditions from other conditions + const textConditions = []; + const nonTextConditions = []; + + for (const condition of conditions) { + switch (condition.field) { + case 'title': + case 'description': + case 'author': + case 'tags': + case 'keyword': + // These can be handled by FTS directly + textConditions.push(condition); + break; + default: + // These need additional WHERE clauses + nonTextConditions.push(condition); + break; + } + } + + // Build FTS MATCH query + let ftsQueryParts = []; + + for (const condition of textConditions) { + let fieldPrefix = ''; + + // Add field-specific prefix if available + if (condition.field !== 'keyword') { + fieldPrefix = `${condition.field}:`; + } + + // Add wildcard for partial matching if not already present + let value = condition.value.trim(); + if (!value.endsWith('*')) { + value = `${value}*`; + } + + ftsQueryParts.push(`${fieldPrefix}${value}`); + } + + // If no text conditions, use a match-all query + const ftsQuery = ftsQueryParts.length > 0 + ? ftsQueryParts.join(operator === 'AND' ? ' AND ' : ' OR ') + : '*'; + + // Build additional WHERE clauses for non-text conditions + let whereClauseParts = []; + const params = []; + + for (const condition of nonTextConditions) { + switch (condition.field) { + case 'date': + const dateOperator = condition.operator === 'after' ? '>' : + condition.operator === 'before' ? '<' : '='; + whereClauseParts.push(`date ${dateOperator} date(?)`); + params.push(condition.value); + break; + + case 'level': + whereClauseParts.push(`level = ?`); + params.push(condition.value); + break; + + case 'logsource': + whereClauseParts.push(`logsource LIKE ?`); + params.push(`%${condition.subfield}%${condition.value}%`); + break; + + case 'id': + whereClauseParts.push(`rule_id = ?`); + params.push(condition.value); + break; + } + } + + // Combine WHERE clauses + const whereClause = whereClauseParts.length > 0 + ? whereClauseParts.join(operator === 'AND' ? ' AND ' : ' OR ') + : ''; + + return { ftsQuery, whereClause, params }; +} + +/** + * Build the SQL query for complex search based on parsed conditions + * + * @param {Object} parsedQuery - The parsed query object + * @param {number} limit - Results limit + * @param {number} offset - Results offset + * @returns {Object} Object with SQL query, count query, and parameters + */ +function buildComplexSqlQuery(parsedQuery, limit, offset) { + const { conditions, operator } = parsedQuery; + const params = []; + + // Start building the primary table selection + let sqlSelectPart = ` + SELECT DISTINCT r.id as rule_id, + (SELECT param_value FROM rule_parameters WHERE rule_id = r.id AND param_name = 'title' LIMIT 1) as title + FROM sigma_rules r + `; + + // Build WHERE clause based on conditions + let whereClauses = []; + let joinIdx = 0; + + for (const condition of conditions) { + let whereClause = ''; + + switch (condition.field) { + case 'title': + joinIdx++; + whereClause = `EXISTS ( + SELECT 1 FROM rule_parameters p${joinIdx} + WHERE p${joinIdx}.rule_id = r.id + AND p${joinIdx}.param_name = 'title' + AND INSTR(LOWER(p${joinIdx}.param_value), LOWER(?)) > 0 + )`; + params.push(condition.value); + break; + + case 'description': + joinIdx++; + whereClause = `EXISTS ( + SELECT 1 FROM rule_parameters p${joinIdx} + WHERE p${joinIdx}.rule_id = r.id + AND p${joinIdx}.param_name = 'description' + AND INSTR(LOWER(p${joinIdx}.param_value), LOWER(?)) > 0 + )`; + params.push(condition.value); + break; + + case 'logsource': + joinIdx++; + whereClause = `EXISTS ( + SELECT 1 FROM rule_parameters p${joinIdx} + WHERE p${joinIdx}.rule_id = r.id + AND p${joinIdx}.param_name = 'logsource' + AND INSTR(LOWER(p${joinIdx}.param_value), LOWER('"${condition.subfield}":"${condition.value}"')) > 0 + )`; + break; + + case 'tags': + joinIdx++; + whereClause = `EXISTS ( + SELECT 1 FROM rule_parameters p${joinIdx} + WHERE p${joinIdx}.rule_id = r.id + AND p${joinIdx}.param_name = 'tags' + AND INSTR(LOWER(p${joinIdx}.param_value), LOWER(?)) > 0 + )`; + params.push(condition.value); + break; + + case 'date': + joinIdx++; + const dateOperator = condition.operator === 'after' ? '>' : + condition.operator === 'before' ? '<' : '='; + whereClause = `r.date ${dateOperator} date(?)`; + params.push(condition.value); + break; + + case 'author': + joinIdx++; + whereClause = `EXISTS ( + SELECT 1 FROM rule_parameters p${joinIdx} + WHERE p${joinIdx}.rule_id = r.id + AND p${joinIdx}.param_name = 'author' + AND INSTR(LOWER(p${joinIdx}.param_value), LOWER(?)) > 0 + )`; + params.push(condition.value); + break; + + case 'level': + joinIdx++; + whereClause = `EXISTS ( + SELECT 1 FROM rule_parameters p${joinIdx} + WHERE p${joinIdx}.rule_id = r.id + AND p${joinIdx}.param_name = 'level' + AND LOWER(p${joinIdx}.param_value) = LOWER(?) + )`; + params.push(condition.value); + break; + + case 'id': + whereClause = `LOWER(r.id) = LOWER(?)`; + params.push(condition.value); + break; + + case 'keyword': + default: + // Default to searching in title + joinIdx++; + whereClause = `EXISTS ( + SELECT 1 FROM rule_parameters p${joinIdx} + WHERE p${joinIdx}.rule_id = r.id + AND p${joinIdx}.param_name = 'title' + AND INSTR(LOWER(p${joinIdx}.param_value), LOWER(?)) > 0 + )`; + params.push(condition.value); + break; + } + + if (whereClause) { + whereClauses.push(whereClause); + } + } + + // Combine the WHERE clauses with the appropriate operator + let whereStatement = ''; + if (whereClauses.length > 0) { + const combiner = operator === 'AND' ? ' AND ' : ' OR '; + whereStatement = `WHERE ${whereClauses.join(combiner)}`; + } + + // Complete queries + const sqlQuery = ` + ${sqlSelectPart} + ${whereStatement} + ORDER BY rule_id + LIMIT ? OFFSET ? + `; + + const sqlCountQuery = ` + SELECT COUNT(DISTINCT r.id) as count + FROM sigma_rules r + ${whereStatement} + `; + + // Add pagination parameters + params.push(limit); + params.push(offset); + + return { sqlQuery, sqlCountQuery, params }; +} + /** * Debug function to retrieve detailed information about a rule's content * Useful for diagnosing issues with rule retrieval and content parsing @@ -579,7 +1197,11 @@ module.exports = { getAllRuleIds, findRuleById, searchRules, + searchRulesFTS, + searchRulesComplex, + searchRulesComplexFTS, debugRuleContent, getRuleYamlContent, - getStatsFromDatabase + getStatsFromDatabase, + checkFtsAvailable }; \ No newline at end of file