lotte/fylgja
0
Code Issues 5 Pull requests Projects Releases Packages Wiki Activity Actions

update search command to use FTS5 SQLite table for complex searches

Browse source
This commit is contained in:
Charlotte Croce 2025-04-18 14:51:43 -04:00
parent d839089153
commit 167829704a
8 changed files with 1359 additions and 267 deletions
Download patch file Download diff file Expand all files Collapse all files

88
src/handlers/config/config_handler.js
View file

@ -1,88 +0,0 @@
//
// config_handler.js
// handle the /sigma-config command
//
const util = require('util');
const { exec } = require('child_process');
const { SIGMA_CLI_PATH } = require('../../config/constants');
const { loadConfig, updateConfig } = require('../../config/config-manager');
const { updateSigmaDatabase } = require('../../services/sigma/sigma_repository_service');
const logger = require('../../utils/logger');
// Promisify exec for async/await usage
const execPromise = util.promisify(exec);
module.exports = (app) => {
app.command('/sigma-config', async ({ command, ack, respond }) => {
await ack();
logger.info(`Sigma config command received: ${command.text}`);
const args = command.text.split(' ');
if (args.length === 0 || args[0] === '') {
// Display current configuration
const config = loadConfig();
logger.info('Displaying current configuration');
await respond(`Current configuration:\nSIEM: ${config.siem}\nLanguage: ${config.lang}\nOutput: ${config.output}`);
return;
}
const configType = args[0];
if (configType === 'update') {
logger.info('Starting database update from command');
try {
await respond('Updating Sigma database... This may take a moment.');
await updateSigmaDatabase();
logger.info('Database update completed from command');
await respond('Sigma database updated successfully');
} catch (error) {
logger.error(`Database update failed: ${error.message}`);
await respond(`Error updating Sigma database: ${error.message}`);
}
return;
}
if (args.length < 2) {
logger.warn(`Invalid config command format: ${command.text}`);
await respond(`Invalid command format. Usage: /sigma-config ${configType} [value]`);
return;
}
const configValue = args[1];
const config = loadConfig();
if (configType === 'siem') {
// Verify the SIEM backend is installed
logger.info(`Attempting to change SIEM to: ${configValue}`);
try {
await execPromise(`${SIGMA_CLI_PATH} list targets | grep ${configValue}`);
updateConfig('siem', configValue);
logger.info(`SIEM configuration updated to: ${configValue}`);
await respond(`SIEM configuration updated to: ${configValue}`);
} catch (error) {
logger.error(`SIEM backend '${configValue}' not found or not installed`);
await respond(`Error: SIEM backend '${configValue}' not found or not installed. Please install it with: sigma plugin install ${configValue}`);
}
} else if (configType === 'lang') {
logger.info(`Changing language to: ${configValue}`);
updateConfig('lang', configValue);
await respond(`Language configuration updated to: ${configValue}`);
} else if (configType === 'output') {
// Check if output format is supported by the current backend
logger.info(`Attempting to change output format to: ${configValue}`);
try {
await execPromise(`${SIGMA_CLI_PATH} list formats ${config.siem} | grep ${configValue}`);
updateConfig('output', configValue);
logger.info(`Output configuration updated to: ${configValue}`);
await respond(`Output configuration updated to: ${configValue}`);
} catch (error) {
logger.error(`Output format '${configValue}' not supported by SIEM backend '${config.siem}'`);
await respond(`Error: Output format '${configValue}' not supported by SIEM backend '${config.siem}'. Run 'sigma list formats ${config.siem}' to see available formats.`);
}
} else {
logger.warn(`Unknown configuration type: ${configType}`);
await respond(`Unknown configuration type: ${configType}. Available types: siem, lang, output, update`);
}
});
};

291
src/handlers/fylgja_command_handler.js
View file

@ -1,142 +1,209 @@
/** /**
* fylgja_command_handler.js * fylgja_command_handler.js
* *
* Unified command handler for the Fylgja Slack bot. * Main handler for the /fylgja slash command
* Processes natural language commands and routes to appropriate handlers. * Parses natural language commands and routes to appropriate handlers
*/ */
const logger = require('../utils/logger'); const logger = require('../utils/logger');
const { parseCommand } = require('../lang/command_parser');
const { handleError } = require('../utils/error_handler'); const { handleError } = require('../utils/error_handler');
const { handleCommand: handleSigmaSearch, handleComplexSearch } = require('./sigma/sigma_search_handler');
const { handleCommand: handleSigmaDetails } = require('./sigma/sigma_details_handler');
const { handleCommand: handleSigmaStats } = require('./sigma/sigma_stats_handler');
const { handleCommand: handleSigmaCreate } = require('./sigma/sigma_create_handler');
const { handleCommand: handleAlerts } = require('./alerts/alerts_handler');
const { handleCommand: handleCase } = require('./case/case_handler');
const { handleCommand: handleConfig } = require('./config/config_handler');
const { handleCommand: handleStats } = require('./stats/stats_handler');
const FILE_NAME = 'fylgja_command_handler.js'; const FILE_NAME = 'fylgja_command_handler.js';
// Import command handlers
const sigmaDetailsHandler = require('./sigma/sigma_details_handler');
const sigmaSearchHandler = require('./sigma/sigma_search_handler');
const sigmaCreateHandler = require('./sigma/sigma_create_handler');
const sigmaStatsHandler = require('./sigma/sigma_stats_handler');
// Import language processing utilities
const commandParser = require('../lang/command_parser');
/** /**
* Handle the universal fylgja command * Main handler for the /fylgja command
* Parses natural language input and routes to appropriate module handlers
* *
* @param {Object} command - The Slack command object * @param {Object} command - The Slack command object
* @param {Function} respond - Function to send response back to Slack * @param {Function} respond - Function to send response back to Slack
*/ */
const handleCommand = async (command, respond) => { const handleCommand = async (command, respond) => {
try { try {
if (!command || !command.text) { logger.info(`${FILE_NAME}: Received command: ${command.text}`);
logger.warn(`${FILE_NAME}: Empty command received for fylgja`);
await respond({
text: 'Please provide a command. Try `/fylgja help` for available commands.',
response_type: 'ephemeral'
});
return;
}
logger.info(`${FILE_NAME}: Processing fylgja command: ${command.text}`); if (!command.text.trim()) {
logger.warn(`${FILE_NAME}: Empty command received`);
// Parse the natural language command await respond({
const parsedCommand = await commandParser.parseCommand(command.text); text: "Please provide a command. Try `/fylgja help` for usage examples.",
response_type: 'ephemeral'
if (!parsedCommand.success) { });
logger.warn(`${FILE_NAME}: Failed to parse command: ${command.text}`); return;
await respond({ }
text: `I couldn't understand that command. ${parsedCommand.message || ''}`,
response_type: 'ephemeral' // Parse the natural language command
}); const parsedCommand = await parseCommand(command.text);
return; logger.debug(`${FILE_NAME}: Parsed command result: ${JSON.stringify(parsedCommand)}`);
if (!parsedCommand.success) {
logger.warn(`${FILE_NAME}: Command parsing failed: ${parsedCommand.message}`);
await respond({
text: parsedCommand.message || "I couldn't understand that command. Try `/fylgja help` for examples.",
response_type: 'ephemeral'
});
return;
}
// Extract the structured command
const { action, module, params } = parsedCommand.command;
logger.info(`${FILE_NAME}: Routing command - Module: ${module}, Action: ${action}`);
// Route to the appropriate handler based on module and action
switch (module) {
case 'sigma':
await handleSigmaCommand(action, params, command, respond);
break;
case 'alerts':
await handleAlerts(command, respond);
break;
case 'case':
await handleCase(command, respond);
break;
case 'config':
await handleConfig(command, respond);
break;
case 'stats':
await handleStats(command, respond);
break;
case 'help':
await handleHelpCommand(respond);
break;
default:
logger.warn(`${FILE_NAME}: Unknown module: ${module}`);
await respond({
text: `Unknown command module: ${module}. Try \`/fylgja help\` for usage examples.`,
response_type: 'ephemeral'
});
}
} catch (error) {
await handleError(error, `${FILE_NAME}: Command handler`, respond, {
responseType: 'ephemeral'
});
} }
// Route to the appropriate handler based on the parsed command
await routeCommand(parsedCommand.command, command, respond);
} catch (error) {
await handleError(error, `${FILE_NAME}: Command handler`, respond, {
responseType: 'ephemeral'
});
}
}; };
/** /**
* Route the command to the appropriate handler * Handle Sigma-related commands
* *
* @param {Object} parsedCommand - The parsed command object * @param {string} action - The action to perform
* @param {Object} originalCommand - The original Slack command object * @param {Array} params - Command parameters
* @param {Function} respond - Function to send response back to Slack * @param {Object} command - The original Slack command
* @param {Function} respond - Function to send response
*/ */
const routeCommand = async (parsedCommand, originalCommand, respond) => { const handleSigmaCommand = async (action, params, command, respond) => {
const { action, module, params } = parsedCommand; logger.debug(`${FILE_NAME}: Handling Sigma command - Action: ${action}, Params: ${JSON.stringify(params)}`);
// Create a modified command object with the extracted parameters try {
const modifiedCommand = { switch (action) {
...originalCommand, case 'search':
text: params.join(' ') // Update the command object with the keyword parameter
}; command.text = params[0] || '';
await handleSigmaSearch(command, respond);
// Log the routing decision break;
logger.debug(`${FILE_NAME}: Routing command - Action: ${action}, Module: ${module}, Params: ${JSON.stringify(params)}`);
case 'complexSearch':
// Route to the appropriate handler // Update the command object with the complex query
switch (`${module}:${action}`) { command.text = params[0] || '';
case 'sigma:details': await handleComplexSearch(command, respond);
case 'sigma:explain': break;
await sigmaDetailsHandler.handleCommand(modifiedCommand, respond);
break; case 'details':
// Update the command object with the rule ID parameter
case 'sigma:search': command.text = params[0] || '';
await sigmaSearchHandler.handleCommand(modifiedCommand, respond); await handleSigmaDetails(command, respond);
break; break;
case 'sigma:create': case 'stats':
await sigmaCreateHandler.handleCommand(modifiedCommand, respond); await handleSigmaStats(command, respond);
break; break;
case 'sigma:stats': case 'create':
await sigmaStatsHandler.handleCommand(modifiedCommand, respond); // Update the command object with the rule ID parameter
break; command.text = params[0] || '';
await handleSigmaCreate(command, respond);
case 'help:general': break;
await showHelp(respond);
break; default:
logger.warn(`${FILE_NAME}: Unknown Sigma action: ${action}`);
default: await respond({
logger.warn(`${FILE_NAME}: Unknown command combination: ${module}:${action}`); text: `Unknown Sigma action: ${action}. Try \`/fylgja help\` for usage examples.`,
await respond({ response_type: 'ephemeral'
text: `I don't know how to ${action} in ${module}. Try \`/fylgja help\` for available commands.`, });
response_type: 'ephemeral' }
}); } catch (error) {
} await handleError(error, `${FILE_NAME}: Sigma command handler`, respond, {
responseType: 'ephemeral'
});
}
}; };
/** /**
* Show help information * Handle help command
* *
* @param {Function} respond - Function to send response back to Slack * @param {Function} respond - Function to send response
*/ */
const showHelp = async (respond) => { const handleHelpCommand = async (respond) => {
await respond({ try {
text: "Here are some example commands you can use with Fylgja:", const helpText = `
blocks: [ *Fylgja Command Help*
{
type: "section", *Basic Commands:*
text: { • \`/fylgja search <keyword>\` - Search for Sigma rules by keyword
type: "mrkdwn", • \`/fylgja details <rule_id>\` - Get details about a specific Sigma rule
text: "*Fylgja Commands*\nHere are some example commands you can use:" • \`/fylgja stats\` - Get statistics about Sigma rules database
}
}, *Advanced Search Commands:*
{ • \`/fylgja search sigma rules where title contains "ransomware"\` - Search by title
type: "section", • \`/fylgja find rules where tags include privilege_escalation\` - Search by tags
text: { • \`/fylgja search rules where logsource.category == "process_creation"\` - Search by log source
type: "mrkdwn", • \`/fylgja find rules where modified after 2024-01-01\` - Search by modification date
text: "• `/fylgja explain rule from sigma where id=<rule_id>`\n• `/fylgja search sigma for <query>`\n• `/fylgja create rule in sigma with <parameters>`\n• `/fylgja show stats for sigma`" • \`/fylgja search where level is "high" and tags include "attack.t1055"\` - Combined search
}
} *Supported Conditions:*
], • Title: \`title contains "text"\`
response_type: 'ephemeral' • Description: \`description contains "text"\`
}); • Log Source: \`logsource.category == "value"\`, \`logsource.product == "value"\`
• Tags: \`tags include "value"\`
• Dates: \`modified after YYYY-MM-DD\`, \`modified before YYYY-MM-DD\`
• Author: \`author is "name"\`
• Level: \`level is "high"\`
*Logical Operators:*
• AND: \`condition1 AND condition2\`
• OR: \`condition1 OR condition2\`
*Pagination:*
• Add \`page=N\` to see page N of results
• Add \`limit=N\` to change number of results per page
For more information, visit the Fylgja documentation.
`;
await respond({
text: helpText,
response_type: 'ephemeral'
});
} catch (error) {
await handleError(error, `${FILE_NAME}: Help command handler`, respond, {
responseType: 'ephemeral'
});
}
}; };
module.exports = { module.exports = {
handleCommand handleCommand
}; };

168
src/handlers/sigma/sigma_search_handler.js
View file

@ -3,11 +3,11 @@
* *
* Handles Sigma rule search requests from Slack commands * Handles Sigma rule search requests from Slack commands
*/ */
const { searchSigmaRules } = require('../../services/sigma/sigma_search_service');
const { searchSigmaRules, searchSigmaRulesComplex } = require('../../services/sigma/sigma_search_service');
const logger = require('../../utils/logger'); const logger = require('../../utils/logger');
const { handleError } = require('../../utils/error_handler'); const { handleError } = require('../../utils/error_handler');
const { getSearchResultBlocks } = require('../../blocks/sigma/sigma_search_results_block'); const { getSearchResultBlocks } = require('../../blocks/sigma/sigma_search_results_block');
const { getFileName } = require('../../utils/file_utils'); const { getFileName } = require('../../utils/file_utils');
const FILE_NAME = getFileName(__filename); const FILE_NAME = getFileName(__filename);
@ -24,18 +24,18 @@ const MAX_RESULTS_THRESHOLD = 99;
const handleCommand = async (command, respond) => { const handleCommand = async (command, respond) => {
try { try {
logger.debug(`${FILE_NAME}: Processing sigma-search command: ${JSON.stringify(command.text)}`); logger.debug(`${FILE_NAME}: Processing sigma-search command: ${JSON.stringify(command.text)}`);
if (!command || !command.text) { if (!command || !command.text) {
logger.warn(`${FILE_NAME}: Empty command received for sigma-search`); logger.warn(`${FILE_NAME}: Empty command received for sigma-search`);
await respond('Invalid command. Usage: /sigma-search [keyword]'); await respond('Invalid command. Usage: /sigma-search [keyword]');
return; return;
} }
// Extract search keyword and check for pagination parameters // Extract search keyword and check for pagination parameters
let keyword = command.text.trim(); let keyword = command.text.trim();
let page = 1; let page = 1;
let pageSize = MAX_RESULTS_PER_PAGE; let pageSize = MAX_RESULTS_PER_PAGE;
// Check for pagination format: keyword page=X // Check for pagination format: keyword page=X
const pagingMatch = keyword.match(/(.+)\s+page=(\d+)$/i); const pagingMatch = keyword.match(/(.+)\s+page=(\d+)$/i);
if (pagingMatch) { if (pagingMatch) {
@ -43,7 +43,7 @@ const handleCommand = async (command, respond) => {
page = parseInt(pagingMatch[2], 10) || 1; page = parseInt(pagingMatch[2], 10) || 1;
logger.debug(`${FILE_NAME}: Detected pagination request: "${keyword}" page ${page}`); logger.debug(`${FILE_NAME}: Detected pagination request: "${keyword}" page ${page}`);
} }
// Check for page size format: keyword limit=X // Check for page size format: keyword limit=X
const limitMatch = keyword.match(/(.+)\s+limit=(\d+)$/i); const limitMatch = keyword.match(/(.+)\s+limit=(\d+)$/i);
if (limitMatch) { if (limitMatch) {
@ -53,29 +53,27 @@ const handleCommand = async (command, respond) => {
pageSize = Math.min(Math.max(pageSize, 1), 100); pageSize = Math.min(Math.max(pageSize, 1), 100);
logger.debug(`${FILE_NAME}: Detected page size request: "${keyword}" limit ${pageSize}`); logger.debug(`${FILE_NAME}: Detected page size request: "${keyword}" limit ${pageSize}`);
} }
if (!keyword) { if (!keyword) {
logger.warn(`${FILE_NAME}: Missing keyword in sigma-search command`); logger.warn(`${FILE_NAME}: Missing keyword in sigma-search command`);
await respond('Invalid command: missing keyword. Usage: /sigma-search [keyword]'); await respond('Invalid command: missing keyword. Usage: /sigma-search [keyword]');
return; return;
} }
logger.info(`${FILE_NAME}: Searching for rules with keyword: ${keyword} (page ${page}, size ${pageSize})`); logger.info(`${FILE_NAME}: Searching for rules with keyword: ${keyword} (page ${page}, size ${pageSize})`);
logger.debug(`${FILE_NAME}: Search keyword length: ${keyword.length}`); logger.debug(`${FILE_NAME}: Search keyword length: ${keyword.length}`);
await respond({ await respond({
text: 'Searching for rules... This may take a moment.', text: 'Searching for rules... This may take a moment.',
response_type: 'ephemeral' response_type: 'ephemeral'
}); });
// Search for rules using the service function with pagination // Search for rules using the service function with pagination
const searchResult = await searchSigmaRules(keyword, page, pageSize); const searchResult = await searchSigmaRules(keyword, page, pageSize);
logger.debug(`${FILE_NAME}: Search result status: ${searchResult.success}`); logger.debug(`${FILE_NAME}: Search result status: ${searchResult.success}`);
logger.debug(`${FILE_NAME}: Found ${searchResult.results?.length || 0} results out of ${searchResult.pagination?.totalResults || 0} total matches`); logger.debug(`${FILE_NAME}: Found ${searchResult.results?.length || 0} results out of ${searchResult.pagination?.totalResults || 0} total matches`);
logger.debug(`${FILE_NAME}: About to generate blocks for search results`); logger.debug(`${FILE_NAME}: About to generate blocks for search results`);
if (!searchResult.success) { if (!searchResult.success) {
logger.error(`${FILE_NAME}: Search failed: ${searchResult.message}`); logger.error(`${FILE_NAME}: Search failed: ${searchResult.message}`);
await respond({ await respond({
@ -84,18 +82,17 @@ const handleCommand = async (command, respond) => {
}); });
return; return;
} }
// Get total count for validation // Get total count for validation
const totalCount = searchResult.pagination?.totalResults || 0; const totalCount = searchResult.pagination?.totalResults || 0;
// Check if search returned too many results // Check if search returned too many results
if (totalCount > MAX_RESULTS_THRESHOLD) { if (totalCount > MAX_RESULTS_THRESHOLD) {
logger.warn(`${FILE_NAME}: Search for "${keyword}" returned too many results (${totalCount}), displaying first page with warning`); logger.warn(`${FILE_NAME}: Search for "${keyword}" returned too many results (${totalCount}), displaying first page with warning`);
// Continue processing but add a notification // Continue processing but add a notification
searchResult.tooManyResults = true; searchResult.tooManyResults = true;
} }
if (!searchResult.results || searchResult.results.length === 0) { if (!searchResult.results || searchResult.results.length === 0) {
if (totalCount > 0) { if (totalCount > 0) {
logger.warn(`${FILE_NAME}: No rules found on page ${page} for "${keyword}", but ${totalCount} total matches exist`); logger.warn(`${FILE_NAME}: No rules found on page ${page} for "${keyword}", but ${totalCount} total matches exist`);
@ -112,16 +109,14 @@ const handleCommand = async (command, respond) => {
} }
return; return;
} }
// Generate blocks with pagination support // Generate blocks with pagination support
let blocks; let blocks;
try { try {
logger.debug(`${FILE_NAME}: Calling getSearchResultBlocks with ${searchResult.results.length} results`); logger.debug(`${FILE_NAME}: Calling getSearchResultBlocks with ${searchResult.results.length} results`);
// If we have too many results, add a warning block at the beginning // If we have too many results, add a warning block at the beginning
if (searchResult.tooManyResults) { if (searchResult.tooManyResults) {
blocks = getSearchResultBlocks(keyword, searchResult.results, searchResult.pagination); blocks = getSearchResultBlocks(keyword, searchResult.results, searchResult.pagination);
// Insert warning at the beginning of blocks (after the header) // Insert warning at the beginning of blocks (after the header)
blocks.splice(1, 0, { blocks.splice(1, 0, {
"type": "section", "type": "section",
@ -133,7 +128,6 @@ const handleCommand = async (command, respond) => {
} else { } else {
blocks = getSearchResultBlocks(keyword, searchResult.results, searchResult.pagination); blocks = getSearchResultBlocks(keyword, searchResult.results, searchResult.pagination);
} }
logger.debug(`${FILE_NAME}: Successfully generated ${blocks?.length || 0} blocks`); logger.debug(`${FILE_NAME}: Successfully generated ${blocks?.length || 0} blocks`);
} catch (blockError) { } catch (blockError) {
// Use error handler for block generation errors // Use error handler for block generation errors
@ -143,19 +137,19 @@ const handleCommand = async (command, respond) => {
}); });
return; return;
} }
// Add debug log before sending response // Add debug log before sending response
logger.debug(`${FILE_NAME}: About to send response with ${blocks?.length || 0} blocks`); logger.debug(`${FILE_NAME}: About to send response with ${blocks?.length || 0} blocks`);
// Determine if this should be visible to everyone or just the user // Determine if this should be visible to everyone or just the user
const isEphemeral = totalCount > 20; const isEphemeral = totalCount > 20;
// Respond with the search results // Respond with the search results
await respond({ await respond({
blocks: blocks, blocks: blocks,
response_type: isEphemeral ? 'ephemeral' : 'in_channel' response_type: isEphemeral ? 'ephemeral' : 'in_channel'
}); });
// Add debug log after sending response // Add debug log after sending response
logger.debug(`${FILE_NAME}: Response sent successfully`); logger.debug(`${FILE_NAME}: Response sent successfully`);
} catch (error) { } catch (error) {
@ -166,6 +160,128 @@ const handleCommand = async (command, respond) => {
} }
}; };
/**
* Handle the complex search command for Sigma rules
* Processes advanced search queries with multiple conditions
*
* @param {Object} command - The Slack command object
* @param {Function} respond - Function to send response back to Slack
*/
const handleComplexSearch = async (command, respond) => {
try {
logger.debug(`${FILE_NAME}: Processing complex search command: ${JSON.stringify(command.text)}`);
if (!command || !command.text) {
logger.warn(`${FILE_NAME}: Empty command received for complex search`);
await respond('Invalid command. Usage: /sigma-search where [conditions]');
return;
}
// Extract query string
let queryString = command.text.trim();
let page = 1;
let pageSize = MAX_RESULTS_PER_PAGE;
// Check for pagination format: query page=X
const pagingMatch = queryString.match(/(.+)\s+page=(\d+)$/i);
if (pagingMatch) {
queryString = pagingMatch[1].trim();
page = parseInt(pagingMatch[2], 10) || 1;
logger.debug(`${FILE_NAME}: Detected pagination request in complex search: page ${page}`);
}
// Check for page size format: query limit=X
const limitMatch = queryString.match(/(.+)\s+limit=(\d+)$/i);
if (limitMatch) {
queryString = limitMatch[1].trim();
pageSize = parseInt(limitMatch[2], 10) || MAX_RESULTS_PER_PAGE;
// Ensure the page size is within reasonable limits
pageSize = Math.min(Math.max(pageSize, 1), 100);
logger.debug(`${FILE_NAME}: Detected page size request in complex search: limit ${pageSize}`);
}
logger.info(`${FILE_NAME}: Performing complex search with query: ${queryString}`);
await respond({
text: 'Processing complex search query... This may take a moment.',
response_type: 'ephemeral'
});
// Perform the complex search
const searchResult = await searchSigmaRulesComplex(queryString, page, pageSize);
if (!searchResult.success) {
logger.error(`${FILE_NAME}: Complex search failed: ${searchResult.message}`);
await respond({
text: `Search failed: ${searchResult.message}`,
response_type: 'ephemeral'
});
return;
}
// Check if we have results
if (!searchResult.results || searchResult.results.length === 0) {
logger.warn(`${FILE_NAME}: No rules found matching complex query criteria`);
await respond({
text: `No rules found matching the specified criteria.`,
response_type: 'ephemeral'
});
return;
}
// Generate blocks with pagination support
let blocks;
try {
// Use the standard search result blocks but with a modified header
blocks = getSearchResultBlocks(
`Complex Query: ${queryString}`,
searchResult.results,
searchResult.pagination
);
// Replace the header to indicate it's a complex search
if (blocks && blocks.length > 0) {
blocks[0] = {
type: "header",
text: {
type: "plain_text",
text: `Sigma Rule Search Results - Query`,
emoji: true
}
};
// Add a description of the search criteria
blocks.splice(1, 0, {
type: "section",
text: {
type: "mrkdwn",
text: `*Query:* \`${queryString}\``
}
});
}
} catch (blockError) {
await handleError(blockError, `${FILE_NAME}: Complex search block generation`, respond, {
responseType: 'ephemeral',
customMessage: `Error generating results view: ${blockError.message}`
});
return;
}
// Respond with the search results
await respond({
blocks: blocks,
response_type: 'ephemeral' // Complex searches are usually more specific to the user
});
logger.info(`${FILE_NAME}: Complex search response sent successfully with ${searchResult.results.length} results`);
} catch (error) {
await handleError(error, `${FILE_NAME}: Complex search handler`, respond, {
responseType: 'ephemeral'
});
}
};
module.exports = { module.exports = {
handleCommand handleCommand,
handleComplexSearch
}; };

17
src/lang/command_patterns.js
View file

@ -30,21 +30,14 @@ const commandPatterns = [
module: 'sigma', module: 'sigma',
params: [2] // rule ID is in capturing group 2 params: [2] // rule ID is in capturing group 2
}, },
// Sigma search patterns // Sigma search patterns
{ {
name: 'sigma-search', name: 'sigma-search',
regex: /^(search|find|look\s+for)\s+(rules|detections)?\s*(in|from)?\s*sigma\s+(for|where|with)?\s+(.+)$/i, regex: /^(search|find)\s+(sigma\s+)?(rules|detections)?\s*(where|with)\s+(.+)$/i,
action: 'search', action: 'complexSearch',
module: 'sigma', module: 'sigma',
params: [5] // search query is in capturing group 5 params: [5] // complex query conditions in capturing group 5
},
{
name: 'sigma-search-simple',
regex: /^(search|find)\s+(.+)$/i,
action: 'search',
module: 'sigma',
params: [2] // search query is in capturing group 2
}, },
// Sigma create patterns // Sigma create patterns
@ -55,7 +48,7 @@ const commandPatterns = [
module: 'sigma', module: 'sigma',
params: [2] // rule ID is in capturing group 2 params: [2] // rule ID is in capturing group 2
}, },
// Sigma stats patterns // Sigma stats patterns
{ {
name: 'sigma-stats-first', name: 'sigma-stats-first',

197
src/lang/query_parser.js Normal file
View file

@ -0,0 +1,197 @@
/**
* query_parser.js
*
* Utility to parse complex search queries for Sigma rules
* Handles conditions like title contains "X", tags include "Y", etc.
*/
const logger = require('../utils/logger');
const FILE_NAME = 'query_parser.js';
/**
* Parse a complex query string into structured search parameters
* Supports conditions like:
* - title contains "ransomware"
* - logsource.category == "process_creation"
* - tags include privilege_escalation
* - modified after 2024-01-01
* - author is "John Doe"
*
* Also supports logical operators:
* - AND, and
* - OR, or
*
* @param {string} queryString - The complex query string to parse
* @returns {Object} Structured search parameters
*/
function parseComplexQuery(queryString) {
try {
logger.debug(`${FILE_NAME}: Parsing complex query: ${queryString}`);
if (!queryString || typeof queryString !== 'string') {
logger.warn(`${FILE_NAME}: Invalid query string`);
return { valid: false, error: 'Invalid query string' };
}
// Initialize the result object
const result = {
valid: true,
conditions: [],
operator: 'AND' // Default to AND for multiple conditions
};
// Check for explicit logical operators
if (/ AND /i.test(queryString)) {
result.operator = 'AND';
// Split by AND and parse each part
const parts = queryString.split(/ AND /i);
for (const part of parts) {
const condition = parseCondition(part.trim());
if (condition) {
result.conditions.push(condition);
}
}
} else if (/ OR /i.test(queryString)) {
result.operator = 'OR';
// Split by OR and parse each part
const parts = queryString.split(/ OR /i);
for (const part of parts) {
const condition = parseCondition(part.trim());
if (condition) {
result.conditions.push(condition);
}
}
} else {
// Single condition
const condition = parseCondition(queryString.trim());
if (condition) {
result.conditions.push(condition);
}
}
// If no valid conditions found, mark as invalid
if (result.conditions.length === 0) {
result.valid = false;
result.error = 'No valid search conditions found';
}
logger.debug(`${FILE_NAME}: Parsed query result: ${JSON.stringify(result)}`);
return result;
} catch (error) {
logger.error(`${FILE_NAME}: Error parsing complex query: ${error.message}`);
return {
valid: false,
error: `Error parsing query: ${error.message}`
};
}
}
/**
* Parse a single condition from the query string
*
* @param {string} conditionStr - The condition string to parse
* @returns {Object|null} Parsed condition object or null if invalid
*/
function parseCondition(conditionStr) {
logger.debug(`${FILE_NAME}: Parsing condition: ${conditionStr}`);
// Define regex patterns for different condition types
const patterns = [
// title contains "value"
{
regex: /^(title|name)\s+(contains|has|like|includes)\s+"?([^"]+)"?$/i,
handler: (matches) => ({
field: 'title',
operator: 'contains',
value: matches[3].trim()
})
},
// description contains "value"
{
regex: /^(description|desc)\s+(contains|has|like|includes)\s+"?([^"]+)"?$/i,
handler: (matches) => ({
field: 'description',
operator: 'contains',
value: matches[3].trim()
})
},
// logsource.category == "value" or logsource.category = "value"
{
regex: /^logsource\.(\w+)\s*(==|=|equals?)\s*"?([^"]+)"?$/i,
handler: (matches) => ({
field: 'logsource',
subfield: matches[1].toLowerCase(),
operator: 'equals',
value: matches[3].trim()
})
},
// tags include "value" or tag contains "value"
{
regex: /^tags?\s+(includes?|contains|has)\s+"?([^"]+)"?$/i,
handler: (matches) => ({
field: 'tags',
operator: 'contains',
value: matches[2].trim()
})
},
// modified after YYYY-MM-DD
{
regex: /^(modified|updated|created|date)\s+(after|before|on|since)\s+"?(\d{4}-\d{2}-\d{2})"?$/i,
handler: (matches) => ({
field: 'date',
type: matches[1].toLowerCase(),
operator: matches[2].toLowerCase(),
value: matches[3].trim()
})
},
// author is "value" or author = "value"
{
regex: /^(author|creator)\s+(is|equals?|==|=)\s+"?([^"]+)"?$/i,
handler: (matches) => ({
field: 'author',
operator: 'equals',
value: matches[3].trim()
})
},
// level is "value" or level = "value"
{
regex: /^(level|severity)\s+(is|equals?|==|=)\s+"?([^"]+)"?$/i,
handler: (matches) => ({
field: 'level',
operator: 'equals',
value: matches[3].trim()
})
},
// id is "value" or id = "value"
{
regex: /^(id|identifier)\s+(is|equals?|==|=)\s+"?([^"]+)"?$/i,
handler: (matches) => ({
field: 'id',
operator: 'equals',
value: matches[3].trim()
})
}
];
// Try each pattern
for (const pattern of patterns) {
const matches = conditionStr.match(pattern.regex);
if (matches) {
return pattern.handler(matches);
}
}
// If we get here, no patterns matched
logger.warn(`${FILE_NAME}: No pattern matched condition: ${conditionStr}`);
// Default to simple keyword search if no specific pattern matches
return {
field: 'keyword',
operator: 'contains',
value: conditionStr.trim()
};
}
module.exports = {
parseComplexQuery
};

111
src/services/sigma/sigma_search_service.js
View file

@ -1,14 +1,15 @@
/** /**
* sigma_search_service.js * sigma_search_service.js
* *
* This service provides functionality for searching Sigma rules by keywords. * This service provides functionality for searching Sigma rules by keywords and complex queries.
* It processes search results and returns them in a structured format. * It processes search results and returns them in a structured format.
* Supports pagination for large result sets. * Supports pagination for large result sets.
*/ */
const { searchRules } = require('../../sigma_db/sigma_db_queries');
const { searchRules, searchRulesComplex } = require('../../sigma_db/sigma_db_queries');
const { parseComplexQuery } = require('../../lang/query_parser');
const logger = require('../../utils/logger'); const logger = require('../../utils/logger');
const { convertSigmaRule } = require('./sigma_converter_service'); const { convertSigmaRule } = require('./sigma_converter_service');
const { getFileName } = require('../../utils/file_utils'); const { getFileName } = require('../../utils/file_utils');
const FILE_NAME = getFileName(__filename); const FILE_NAME = getFileName(__filename);
@ -153,6 +154,109 @@ async function searchSigmaRules(keyword, page = 1, pageSize = 10) {
} }
} }
/**
* Searches for Sigma rules using complex query conditions
* Supports filtering by title, logsource, tags, dates, and more
*
* @param {string} queryString - The complex query string to parse
* @param {number} page - Page number (1-based index, default: 1)
* @param {number} pageSize - Number of results per page (default: 10)
* @returns {Promise<Object>} Result object with success flag and processed results
*/
async function searchSigmaRulesComplex(queryString, page = 1, pageSize = 10) {
if (!queryString || typeof queryString !== 'string') {
logger.warn(`${FILE_NAME}: Cannot perform complex search: Missing or invalid query string`);
return {
success: false,
message: 'Missing or invalid complex query'
};
}
// Validate pagination parameters
if (typeof page !== 'number' || page < 1) {
logger.warn(`${FILE_NAME}: Invalid page number: ${page}, defaulting to 1`);
page = 1;
}
if (typeof pageSize !== 'number' || pageSize < 1 || pageSize > 100) {
logger.warn(`${FILE_NAME}: Invalid page size: ${pageSize}, defaulting to 10`);
pageSize = 10;
}
// Calculate the offset based on page number
const offset = (page - 1) * pageSize;
logger.info(`${FILE_NAME}: Performing complex search with query: "${queryString}" (page ${page}, size ${pageSize})`);
try {
// Parse the complex query string
const parsedQuery = parseComplexQuery(queryString);
if (!parsedQuery.valid) {
logger.warn(`${FILE_NAME}: Invalid complex query: ${parsedQuery.error}`);
return {
success: false,
message: `Invalid query: ${parsedQuery.error}`
};
}
// Perform the database search with the parsed query
const searchResult = await searchRulesComplex(parsedQuery, pageSize, offset);
// Defensive handling of possible return formats
let allResults = [];
let totalCount = 0;
// Handle search results
if (searchResult) {
if (Array.isArray(searchResult.results)) {
allResults = searchResult.results;
totalCount = searchResult.totalCount || 0;
}
}
if (allResults.length === 0) {
return {
success: true,
results: [],
message: `No rules found matching the complex query criteria`,
pagination: {
currentPage: page,
pageSize: pageSize,
totalPages: Math.ceil(totalCount / pageSize),
totalResults: totalCount,
hasMore: false
}
};
}
// Calculate pagination info
const totalPages = Math.ceil(totalCount / pageSize);
const hasMore = (offset + pageSize) < totalCount;
return {
success: true,
results: allResults,
count: allResults.length,
query: parsedQuery,
pagination: {
currentPage: page,
pageSize: pageSize,
totalPages: totalPages,
totalResults: totalCount,
hasMore: hasMore
}
};
} catch (error) {
logger.error(`${FILE_NAME}: Error in complex search: ${error.message}`);
logger.debug(`${FILE_NAME}: Error stack: ${error.stack}`);
return {
success: false,
message: `Error performing complex search: ${error.message}`
};
}
}
/** /**
* Enhanced search that returns fully converted rule objects with pagination support * Enhanced search that returns fully converted rule objects with pagination support
* This is a more expensive operation than basic search * This is a more expensive operation than basic search
@ -210,5 +314,6 @@ async function searchAndConvertRules(keyword, page = 1, pageSize = 10) {
module.exports = { module.exports = {
searchSigmaRules, searchSigmaRules,
searchSigmaRulesComplex,
searchAndConvertRules searchAndConvertRules
}; };

130
src/sigma_db/sigma_db_initialize.js
View file

@ -58,41 +58,49 @@ async function initializeDatabase(db) {
return; return;
} }
// Create rules table with basic information // Drop FTS table if exists
const createRulesTableSql = ` db.run('DROP TABLE IF EXISTS rule_search', (err) => {
CREATE TABLE sigma_rules (
id TEXT PRIMARY KEY,
file_path TEXT,
content TEXT,
date DATETIME DEFAULT CURRENT_TIMESTAMP
)
`;
db.run(createRulesTableSql, (err) => {
if (err) { if (err) {
reject(err); reject(err);
return; return;
} }
// Create rule_parameters table for individual parameters // Create rules table with basic information
const createParamsTableSql = ` const createRulesTableSql = `
CREATE TABLE rule_parameters ( CREATE TABLE sigma_rules (
id INTEGER PRIMARY KEY AUTOINCREMENT, id TEXT PRIMARY KEY,
rule_id TEXT, file_path TEXT,
param_name TEXT, content TEXT,
param_value TEXT, date DATETIME DEFAULT CURRENT_TIMESTAMP
param_type TEXT,
FOREIGN KEY (rule_id) REFERENCES sigma_rules(id) ON DELETE CASCADE
) )
`; `;
db.run(createParamsTableSql, (err) => { db.run(createRulesTableSql, (err) => {
if (err) { if (err) {
reject(err); reject(err);
} else { return;
logger.info(`${FILE_NAME}: Database schema initialized`);
resolve();
} }
// Create rule_parameters table for individual parameters
const createParamsTableSql = `
CREATE TABLE rule_parameters (
id INTEGER PRIMARY KEY AUTOINCREMENT,
rule_id TEXT,
param_name TEXT,
param_value TEXT,
param_type TEXT,
FOREIGN KEY (rule_id) REFERENCES sigma_rules(id) ON DELETE CASCADE
)
`;
db.run(createParamsTableSql, (err) => {
if (err) {
reject(err);
} else {
logger.info(`${FILE_NAME}: Database schema initialized`);
resolve();
}
});
}); });
}); });
}); });
@ -100,6 +108,70 @@ async function initializeDatabase(db) {
}); });
} }
// Create FTS5 virtual table for full-text search
async function createFtsTable(db) {
return new Promise((resolve, reject) => {
logger.info(`${FILE_NAME}: Creating FTS5 virtual table for full-text search`);
// Create the FTS5 virtual table
const createFtsTableSql = `
CREATE VIRTUAL TABLE IF NOT EXISTS rule_search USING fts5(
rule_id,
title,
description,
logsource,
tags,
author,
level,
content,
tokenize="unicode61"
);
`;
db.run(createFtsTableSql, (err) => {
if (err) {
logger.error(`${FILE_NAME}: Failed to create FTS5 table: ${err.message}`);
reject(err);
} else {
logger.info(`${FILE_NAME}: FTS5 virtual table created successfully`);
resolve();
}
});
});
}
// Populate FTS table with rule data for full-text search
async function populateFtsTable(db) {
return new Promise((resolve, reject) => {
logger.info(`${FILE_NAME}: Populating FTS5 table with rule data`);
// Insert query that aggregates data from both tables
const populateFtsSql = `
INSERT INTO rule_search(rule_id, title, description, logsource, tags, author, level, content)
SELECT
r.id,
(SELECT param_value FROM rule_parameters WHERE rule_id = r.id AND param_name = 'title' LIMIT 1),
(SELECT param_value FROM rule_parameters WHERE rule_id = r.id AND param_name = 'description' LIMIT 1),
(SELECT param_value FROM rule_parameters WHERE rule_id = r.id AND param_name = 'logsource' LIMIT 1),
(SELECT param_value FROM rule_parameters WHERE rule_id = r.id AND param_name = 'tags' LIMIT 1),
(SELECT param_value FROM rule_parameters WHERE rule_id = r.id AND param_name = 'author' LIMIT 1),
(SELECT param_value FROM rule_parameters WHERE rule_id = r.id AND param_name = 'level' LIMIT 1),
r.content
FROM sigma_rules r
`;
db.run(populateFtsSql, (err) => {
if (err) {
logger.error(`${FILE_NAME}: Failed to populate FTS5 table: ${err.message}`);
reject(err);
} else {
logger.info(`${FILE_NAME}: FTS5 table populated successfully`);
resolve();
}
});
});
}
// Determine if a YAML document is a Sigma rule // Determine if a YAML document is a Sigma rule
function isSigmaRule(doc) { function isSigmaRule(doc) {
// Check for essential Sigma rule properties // Check for essential Sigma rule properties
@ -531,6 +603,12 @@ async function main() {
// Create indexes // Create indexes
await createIndexes(db); await createIndexes(db);
// Create FTS5 table
await createFtsTable(db);
// Populate FTS5 table with rule data
await populateFtsTable(db);
// Close database connection // Close database connection
db.close((err) => { db.close((err) => {
if (err) { if (err) {
@ -556,5 +634,7 @@ if (require.main === module) {
module.exports = { module.exports = {
initializeDatabase, initializeDatabase,
importRules, importRules,
createIndexes createIndexes,
createFtsTable,
populateFtsTable
}; };

624
src/sigma_db/sigma_db_queries.js
View file

@ -198,6 +198,17 @@ async function searchRules(keyword, limit = 10, offset = 0) {
db = await getDbConnection(); db = await getDbConnection();
logger.debug(`${FILE_NAME}: Database connection established for search`); logger.debug(`${FILE_NAME}: Database connection established for search`);
// Use FTS5 for faster searching if available
const ftsAvailable = await checkFtsAvailable(db);
if (ftsAvailable) {
logger.debug(`${FILE_NAME}: Using FTS5 for keyword search`);
return searchRulesFTS(keyword, limit, offset);
}
// If FTS5 is not available, use the legacy search method
logger.debug(`${FILE_NAME}: FTS5 not available, using legacy search method`);
// First get the total count of matching rules (for pagination info) // First get the total count of matching rules (for pagination info)
const countQuery = ` const countQuery = `
SELECT COUNT(*) as count SELECT COUNT(*) as count
@ -264,6 +275,613 @@ async function searchRules(keyword, limit = 10, offset = 0) {
} }
} }
/**
* Check if FTS5 virtual table is available
*
* @param {Object} db - Database connection
* @returns {Promise<boolean>} Whether FTS5 is available
*/
async function checkFtsAvailable(db) {
try {
const result = await new Promise((resolve, reject) => {
db.get("SELECT name FROM sqlite_master WHERE type='table' AND name='rule_search'", (err, row) => {
if (err) {
logger.error(`${FILE_NAME}: Error checking for FTS5 table: ${err.message}`);
reject(err);
} else {
resolve(row !== undefined);
}
});
});
logger.debug(`${FILE_NAME}: FTS5 table availability check: ${result ? 'Available' : 'Not available'}`);
return result;
} catch (error) {
logger.error(`${FILE_NAME}: Error checking FTS availability: ${error.message}`);
return false;
}
}
/**
* Search for Sigma rules using FTS5
* Performs a full-text search and returns matching rules with pagination
*
* @param {string} keyword - The keyword to search for
* @param {number} limit - Maximum number of results to return (default: 10)
* @param {number} offset - Number of results to skip (for pagination, default: 0)
* @returns {Promise<Object>} Object with results array and total count
*/
async function searchRulesFTS(keyword, limit = 10, offset = 0) {
if (!keyword) {
logger.warn(`${FILE_NAME}: Empty search keyword provided for FTS search`);
return { results: [], totalCount: 0 };
}
// Prepare FTS query - add * for prefix matching if not already present
let ftsQuery = keyword.trim();
if (!ftsQuery.endsWith('*')) {
ftsQuery = `${ftsQuery}*`;
}
logger.info(`${FILE_NAME}: Performing FTS search with query: "${ftsQuery}" (limit: ${limit}, offset: ${offset})`);
let db;
try {
db = await getDbConnection();
logger.debug(`${FILE_NAME}: Database connection established for FTS search`);
// First get the total count of matching rules
const countQuery = `
SELECT COUNT(*) as count
FROM rule_search
WHERE rule_search MATCH ?
`;
const countResult = await new Promise((resolve, reject) => {
db.get(countQuery, [ftsQuery], (err, row) => {
if (err) {
logger.error(`${FILE_NAME}: FTS count query error: ${err.message}`);
reject(err);
} else {
resolve(row || { count: 0 });
}
});
});
const totalCount = countResult.count;
logger.debug(`${FILE_NAME}: Total matching rules for FTS query "${ftsQuery}": ${totalCount}`);
// Now get the actual results with pagination
const searchQuery = `
SELECT rule_id, title
FROM rule_search
WHERE rule_search MATCH ?
ORDER BY rank
LIMIT ? OFFSET ?
`;
const results = await new Promise((resolve, reject) => {
db.all(searchQuery, [ftsQuery, limit, offset], (err, rows) => {
if (err) {
logger.error(`${FILE_NAME}: FTS search query error: ${err.message}`);
reject(err);
} else {
logger.debug(`${FILE_NAME}: FTS search query returned ${rows ? rows.length : 0} results`);
resolve(rows || []);
}
});
});
logger.debug(`${FILE_NAME}: FTS search results page for query "${ftsQuery}": ${results.length} matches (page ${Math.floor(offset / limit) + 1})`);
return {
results: results.map(r => ({ id: r.rule_id, title: r.title || r.rule_id })),
totalCount
};
} catch (error) {
logger.error(`${FILE_NAME}: Error in FTS search operation: ${error.message}`);
logger.debug(`${FILE_NAME}: FTS search error stack: ${error.stack}`);
return { results: [], totalCount: 0 };
} finally {
if (db) {
try {
await db.close();
logger.debug(`${FILE_NAME}: Database connection closed after FTS search operation`);
} catch (closeError) {
logger.error(`${FILE_NAME}: Error closing database connection after FTS search: ${closeError.message}`);
}
}
}
}
/**
* Search for Sigma rules using complex query conditions
* Supports filtering by multiple attributes like title, logsource, tags, etc.
*
* @param {Object} parsedQuery - The parsed query object containing conditions and operator
* @param {number} limit - Maximum number of results to return
* @param {number} offset - Number of results to skip (for pagination)
* @returns {Promise<Object>} Object with results array and total count
*/
async function searchRulesComplex(parsedQuery, limit = 10, offset = 0) {
if (!parsedQuery || !parsedQuery.valid) {
logger.warn(`${FILE_NAME}: Invalid query object provided`);
return { results: [], totalCount: 0 };
}
logger.info(`${FILE_NAME}: Performing complex search with ${parsedQuery.conditions.length} conditions (limit: ${limit}, offset: ${offset})`);
let db;
// Declare this at function scope so it's available in the finally block
let usingFts = false;
try {
db = await getDbConnection();
logger.debug(`${FILE_NAME}: Database connection established for complex search`);
// Check if FTS5 is available
const ftsAvailable = await checkFtsAvailable(db);
if (ftsAvailable) {
logger.debug(`${FILE_NAME}: Using FTS5 for complex search`);
// Set flag that we're using FTS
usingFts = true;
// Pass db connection to searchRulesComplexFTS and let that function manage it
const results = await searchRulesComplexFTS(parsedQuery, limit, offset, db);
return results;
}
logger.debug(`${FILE_NAME}: FTS5 not available, using legacy complex search method`);
// Build the SQL query based on the conditions
const { sqlQuery, sqlCountQuery, params } = buildComplexSqlQuery(parsedQuery, limit, offset);
logger.debug(`${FILE_NAME}: Executing complex search SQL: ${sqlQuery}`);
logger.debug(`${FILE_NAME}: Query parameters: ${JSON.stringify(params)}`);
// First get the total count of matching results
const countResult = await new Promise((resolve, reject) => {
db.get(sqlCountQuery, params.slice(0, params.length - 2), (err, row) => {
if (err) {
logger.error(`${FILE_NAME}: Complex search count query error: ${err.message}`);
reject(err);
} else {
resolve(row || { count: 0 });
}
});
});
const totalCount = countResult.count;
logger.debug(`${FILE_NAME}: Total matching rules for complex query: ${totalCount}`);
// Now get the actual results with pagination
const results = await new Promise((resolve, reject) => {
db.all(sqlQuery, params, (err, rows) => {
if (err) {
logger.error(`${FILE_NAME}: Complex search query error: ${err.message}`);
reject(err);
} else {
logger.debug(`${FILE_NAME}: Complex search query returned ${rows ? rows.length : 0} results`);
resolve(rows || []);
}
});
});
// Format the results
const formattedResults = results.map(r => ({
id: r.rule_id,
title: r.title || r.rule_id
}));
logger.debug(`${FILE_NAME}: Returning ${formattedResults.length} results for complex search`);
return {
results: formattedResults,
totalCount
};
} catch (error) {
logger.error(`${FILE_NAME}: Error in complex search operation: ${error.message}`);
logger.debug(`${FILE_NAME}: Complex search error stack: ${error.stack}`);
return { results: [], totalCount: 0 };
} finally {
// IMPORTANT: Only close the db connection if we're not using FTS
// When using FTS, let searchRulesComplexFTS manage the connection
if (db && !usingFts) {
try {
await new Promise((resolve) => db.close(() => resolve()));
logger.debug(`${FILE_NAME}: Database connection closed after complex search operation`);
} catch (closeError) {
logger.error(`${FILE_NAME}: Error closing database after complex search: ${closeError.message}`);
}
}
}
}
/**
* Search for Sigma rules using complex query conditions with FTS5
* Uses the FTS5 virtual table for faster text searching
*
* @param {Object} parsedQuery - The parsed query object
* @param {number} limit - Maximum number of results to return
* @param {number} offset - Number of results to skip (for pagination)
* @param {Object} providedDb - Database connection (optional, will create one if not provided)
* @returns {Promise<Object>} Object with results array and total count
*/
async function searchRulesComplexFTS(parsedQuery, limit = 10, offset = 0, providedDb = null) {
if (!parsedQuery || !parsedQuery.valid) {
logger.warn(`${FILE_NAME}: Invalid query object provided for FTS complex search`);
return { results: [], totalCount: 0 };
}
logger.info(`${FILE_NAME}: Performing complex FTS search with ${parsedQuery.conditions.length} conditions`);
let db;
let shouldCloseDb = false;
try {
// Use provided db connection or create a new one
if (providedDb) {
db = providedDb;
} else {
db = await getDbConnection();
shouldCloseDb = true;
logger.debug(`${FILE_NAME}: Created new database connection for complex FTS search`);
}
// Build FTS query from conditions
const { ftsQuery, whereClause, params } = buildComplexFtsQuery(parsedQuery);
logger.debug(`${FILE_NAME}: FTS query: "${ftsQuery}", additional where: ${whereClause ? whereClause : 'none'}`);
logger.debug(`${FILE_NAME}: Query parameters: ${JSON.stringify(params)}`);
// Build count query
let countQuery;
let countParams;
if (whereClause) {
countQuery = `
SELECT COUNT(*) as count
FROM rule_search
WHERE rule_search MATCH ?
AND ${whereClause}
`;
countParams = [ftsQuery, ...params];
} else {
countQuery = `
SELECT COUNT(*) as count
FROM rule_search
WHERE rule_search MATCH ?
`;
countParams = [ftsQuery];
}
// Get total count
const countResult = await new Promise((resolve, reject) => {
db.get(countQuery, countParams, (err, row) => {
if (err) {
logger.error(`${FILE_NAME}: Complex FTS count query error: ${err.message}`);
reject(err);
} else {
resolve(row || { count: 0 });
}
});
});
const totalCount = countResult.count;
logger.debug(`${FILE_NAME}: Total matching rules for complex FTS query: ${totalCount}`);
// Build results query with pagination
let searchQuery;
let searchParams;
if (whereClause) {
searchQuery = `
SELECT rule_id, title
FROM rule_search
WHERE rule_search MATCH ?
AND ${whereClause}
ORDER BY rank
LIMIT ? OFFSET ?
`;
searchParams = [ftsQuery, ...params, limit, offset];
} else {
searchQuery = `
SELECT rule_id, title
FROM rule_search
WHERE rule_search MATCH ?
ORDER BY rank
LIMIT ? OFFSET ?
`;
searchParams = [ftsQuery, limit, offset];
}
// Get paginated results
const results = await new Promise((resolve, reject) => {
db.all(searchQuery, searchParams, (err, rows) => {
if (err) {
logger.error(`${FILE_NAME}: Complex FTS search query error: ${err.message}`);
reject(err);
} else {
logger.debug(`${FILE_NAME}: Complex FTS search query returned ${rows ? rows.length : 0} results`);
resolve(rows || []);
}
});
});
// Format the results
const formattedResults = results.map(r => ({
id: r.rule_id,
title: r.title || r.rule_id
}));
logger.debug(`${FILE_NAME}: Returning ${formattedResults.length} results for complex FTS search`);
return {
results: formattedResults,
totalCount
};
} catch (error) {
logger.error(`${FILE_NAME}: Error in complex FTS search operation: ${error.message}`);
logger.debug(`${FILE_NAME}: Complex FTS search error stack: ${error.stack}`);
return { results: [], totalCount: 0 };
} finally {
// Only close the database if we created it AND we're not in the middle of a transaction
if (db && shouldCloseDb) {
try {
await db.close();
logger.debug(`${FILE_NAME}: Database connection closed after complex FTS search`);
} catch (closeError) {
logger.error(`${FILE_NAME}: Error closing database after complex FTS search: ${closeError.message}`);
}
}
}
}
/**
* Build FTS query and WHERE clause from parsed query conditions
*
* @param {Object} parsedQuery - The parsed query object
* @returns {Object} Object with FTS query, additional WHERE clause, and parameters
*/
function buildComplexFtsQuery(parsedQuery) {
const { conditions, operator } = parsedQuery;
// Separate text search conditions from other conditions
const textConditions = [];
const nonTextConditions = [];
for (const condition of conditions) {
switch (condition.field) {
case 'title':
case 'description':
case 'author':
case 'tags':
case 'keyword':
// These can be handled by FTS directly
textConditions.push(condition);
break;
default:
// These need additional WHERE clauses
nonTextConditions.push(condition);
break;
}
}
// Build FTS MATCH query
let ftsQueryParts = [];
for (const condition of textConditions) {
let fieldPrefix = '';
// Add field-specific prefix if available
if (condition.field !== 'keyword') {
fieldPrefix = `${condition.field}:`;
}
// Add wildcard for partial matching if not already present
let value = condition.value.trim();
if (!value.endsWith('*')) {
value = `${value}*`;
}
ftsQueryParts.push(`${fieldPrefix}${value}`);
}
// If no text conditions, use a match-all query
const ftsQuery = ftsQueryParts.length > 0
? ftsQueryParts.join(operator === 'AND' ? ' AND ' : ' OR ')
: '*';
// Build additional WHERE clauses for non-text conditions
let whereClauseParts = [];
const params = [];
for (const condition of nonTextConditions) {
switch (condition.field) {
case 'date':
const dateOperator = condition.operator === 'after' ? '>' :
condition.operator === 'before' ? '<' : '=';
whereClauseParts.push(`date ${dateOperator} date(?)`);
params.push(condition.value);
break;
case 'level':
whereClauseParts.push(`level = ?`);
params.push(condition.value);
break;
case 'logsource':
whereClauseParts.push(`logsource LIKE ?`);
params.push(`%${condition.subfield}%${condition.value}%`);
break;
case 'id':
whereClauseParts.push(`rule_id = ?`);
params.push(condition.value);
break;
}
}
// Combine WHERE clauses
const whereClause = whereClauseParts.length > 0
? whereClauseParts.join(operator === 'AND' ? ' AND ' : ' OR ')
: '';
return { ftsQuery, whereClause, params };
}
/**
* Build the SQL query for complex search based on parsed conditions
*
* @param {Object} parsedQuery - The parsed query object
* @param {number} limit - Results limit
* @param {number} offset - Results offset
* @returns {Object} Object with SQL query, count query, and parameters
*/
function buildComplexSqlQuery(parsedQuery, limit, offset) {
const { conditions, operator } = parsedQuery;
const params = [];
// Start building the primary table selection
let sqlSelectPart = `
SELECT DISTINCT r.id as rule_id,
(SELECT param_value FROM rule_parameters WHERE rule_id = r.id AND param_name = 'title' LIMIT 1) as title
FROM sigma_rules r
`;
// Build WHERE clause based on conditions
let whereClauses = [];
let joinIdx = 0;
for (const condition of conditions) {
let whereClause = '';
switch (condition.field) {
case 'title':
joinIdx++;
whereClause = `EXISTS (
SELECT 1 FROM rule_parameters p${joinIdx}
WHERE p${joinIdx}.rule_id = r.id
AND p${joinIdx}.param_name = 'title'
AND INSTR(LOWER(p${joinIdx}.param_value), LOWER(?)) > 0
)`;
params.push(condition.value);
break;
case 'description':
joinIdx++;
whereClause = `EXISTS (
SELECT 1 FROM rule_parameters p${joinIdx}
WHERE p${joinIdx}.rule_id = r.id
AND p${joinIdx}.param_name = 'description'
AND INSTR(LOWER(p${joinIdx}.param_value), LOWER(?)) > 0
)`;
params.push(condition.value);
break;
case 'logsource':
joinIdx++;
whereClause = `EXISTS (
SELECT 1 FROM rule_parameters p${joinIdx}
WHERE p${joinIdx}.rule_id = r.id
AND p${joinIdx}.param_name = 'logsource'
AND INSTR(LOWER(p${joinIdx}.param_value), LOWER('"${condition.subfield}":"${condition.value}"')) > 0
)`;
break;
case 'tags':
joinIdx++;
whereClause = `EXISTS (
SELECT 1 FROM rule_parameters p${joinIdx}
WHERE p${joinIdx}.rule_id = r.id
AND p${joinIdx}.param_name = 'tags'
AND INSTR(LOWER(p${joinIdx}.param_value), LOWER(?)) > 0
)`;
params.push(condition.value);
break;
case 'date':
joinIdx++;
const dateOperator = condition.operator === 'after' ? '>' :
condition.operator === 'before' ? '<' : '=';
whereClause = `r.date ${dateOperator} date(?)`;
params.push(condition.value);
break;
case 'author':
joinIdx++;
whereClause = `EXISTS (
SELECT 1 FROM rule_parameters p${joinIdx}
WHERE p${joinIdx}.rule_id = r.id
AND p${joinIdx}.param_name = 'author'
AND INSTR(LOWER(p${joinIdx}.param_value), LOWER(?)) > 0
)`;
params.push(condition.value);
break;
case 'level':
joinIdx++;
whereClause = `EXISTS (
SELECT 1 FROM rule_parameters p${joinIdx}
WHERE p${joinIdx}.rule_id = r.id
AND p${joinIdx}.param_name = 'level'
AND LOWER(p${joinIdx}.param_value) = LOWER(?)
)`;
params.push(condition.value);
break;
case 'id':
whereClause = `LOWER(r.id) = LOWER(?)`;
params.push(condition.value);
break;
case 'keyword':
default:
// Default to searching in title
joinIdx++;
whereClause = `EXISTS (
SELECT 1 FROM rule_parameters p${joinIdx}
WHERE p${joinIdx}.rule_id = r.id
AND p${joinIdx}.param_name = 'title'
AND INSTR(LOWER(p${joinIdx}.param_value), LOWER(?)) > 0
)`;
params.push(condition.value);
break;
}
if (whereClause) {
whereClauses.push(whereClause);
}
}
// Combine the WHERE clauses with the appropriate operator
let whereStatement = '';
if (whereClauses.length > 0) {
const combiner = operator === 'AND' ? ' AND ' : ' OR ';
whereStatement = `WHERE ${whereClauses.join(combiner)}`;
}
// Complete queries
const sqlQuery = `
${sqlSelectPart}
${whereStatement}
ORDER BY rule_id
LIMIT ? OFFSET ?
`;
const sqlCountQuery = `
SELECT COUNT(DISTINCT r.id) as count
FROM sigma_rules r
${whereStatement}
`;
// Add pagination parameters
params.push(limit);
params.push(offset);
return { sqlQuery, sqlCountQuery, params };
}
/** /**
* Debug function to retrieve detailed information about a rule's content * Debug function to retrieve detailed information about a rule's content
* Useful for diagnosing issues with rule retrieval and content parsing * Useful for diagnosing issues with rule retrieval and content parsing
@ -579,7 +1197,11 @@ module.exports = {
getAllRuleIds, getAllRuleIds,
findRuleById, findRuleById,
searchRules, searchRules,
searchRulesFTS,
searchRulesComplex,
searchRulesComplexFTS,
debugRuleContent, debugRuleContent,
getRuleYamlContent, getRuleYamlContent,
getStatsFromDatabase getStatsFromDatabase,
checkFtsAvailable
}; };