From 2082b844b4528d7120cff8b2cc3e71c200d58b3c Mon Sep 17 00:00:00 2001 From: Charlotte Croce Date: Wed, 16 Apr 2025 18:13:03 -0400 Subject: [PATCH 01/15] update fylgja.example.yml --- fylgja.example.yml | 35 +++++++++++++++++++++++++++++------ 1 file changed, 29 insertions(+), 6 deletions(-) diff --git a/fylgja.example.yml b/fylgja.example.yml index fc66c2c..344c3df 100644 --- a/fylgja.example.yml +++ b/fylgja.example.yml @@ -3,8 +3,8 @@ # Slack settings slack: - bot_token: "xoxb-TOKEN_HERE" - signing_secret: "SIGNING_SECRET_HERE" + bot_token: "xoxb-TOKEN" + signing_secret: "SIGNING_SECRET" # Server settings server: @@ -26,10 +26,33 @@ sigma: url: "https://github.com/SigmaHQ/sigma.git" branch: "main" -# Elastic settings -elastic: - api-endpoint: "http://localhost:5601/api/detection_engine/rules" - elastic-authentication-credentials: "elastic:changeme" +# Elasticsearch settings +elasticsearch: + protocol: "http" + hosts: ["localhost:9200"] + username: "elastic" + password: "changeme" + api_endpoint: "http://localhost:5601/api/detection_engine/rules" + spaces: [ + { + name: "Default", + id: "default", + indexPattern: "logs-*", + emoji: "🔍" + }, + { + name: "space2", + id: "space2", + indexPattern: ["space2-*", "test2-*"], + emoji: "🟢" + }, + { + name: "space3", + id: "space3", + indexPattern: "space3-*", + emoji: "🐧" + } + ] # Logging settings logging: -- 2.39.5 From bfabd6de2ae4bb4096ab2c4e1791a3fd44ae00af Mon Sep 17 00:00:00 2001 From: Charlotte Croce Date: Fri, 18 Apr 2025 13:06:09 -0400 Subject: [PATCH 02/15] refactor elastic api into multiple files --- src/handlers/sigma/sigma_action_handlers.js | 4 +- src/services/elastic/elastic_api_service.js | 83 ---------------- .../elastic_send_rule_to_siem_service.js | 96 +++++++++++++++++++ 3 files changed, 98 insertions(+), 85 deletions(-) create mode 100644 src/services/elastic/elastic_send_rule_to_siem_service.js diff --git a/src/handlers/sigma/sigma_action_handlers.js b/src/handlers/sigma/sigma_action_handlers.js index 49edcd5..81a0445 100644 --- a/src/handlers/sigma/sigma_action_handlers.js +++ b/src/handlers/sigma/sigma_action_handlers.js @@ -12,9 +12,9 @@ const { getYamlViewBlocks } = require('../../blocks/sigma/sigma_view_yaml_block' const { getSearchResultBlocks } = require('../../blocks/sigma/sigma_search_results_block'); const { getConversionResultBlocks } = require('../../blocks/sigma/sigma_conversion_block'); const { getRuleExplanationBlocks } = require('../../blocks/sigma/sigma_details_block'); -const { sendRuleToSiem } = require('../../services/elastic/elastic_api_service'); -const { getSpaceSelectionBlocks } = require('../../blocks/sigma/sigma_space_selection_block'); +const { sendRuleToSiem } = require('../../services/elastic/elastic_send_rule_to_siem_service'); const { getAllSpaces } = require('../../services/elastic/elastic_api_service'); +const { getSpaceSelectionBlocks } = require('../../blocks/sigma/sigma_space_selection_block'); const { SIGMA_CLI_CONFIG, ELASTICSEARCH_CONFIG } = require('../../config/appConfig'); diff --git a/src/services/elastic/elastic_api_service.js b/src/services/elastic/elastic_api_service.js index 965c36b..79107c6 100644 --- a/src/services/elastic/elastic_api_service.js +++ b/src/services/elastic/elastic_api_service.js @@ -49,88 +49,6 @@ const getAllSpaces = () => { return ELASTICSEARCH_CONFIG.spaces || []; }; -/** - * Send a rule to Elasticsearch SIEM in a specific space - * - * @param {Object} rulePayload - The rule payload to send to Elasticsearch - * @param {string} spaceId - The ID of the space to send the rule to - * @returns {Promise} - Object containing success status and response/error information - */ -const sendRuleToSiem = async (rulePayload, spaceId = 'default') => { - logger.info(`${FILE_NAME}: Sending rule to Elasticsearch SIEM in space: ${spaceId}`); - - try { - const elasticConfig = getElasticConfig(spaceId); - const baseApiUrl = elasticConfig.apiEndpoint; - - // Construct space-specific URL if needed - let apiUrl = baseApiUrl; - if (spaceId && spaceId !== 'default') { - // Insert space ID into URL: http://localhost:5601/api/detection_engine/rules - // becomes http://localhost:5601/s/space-id/api/detection_engine/rules - const urlParts = baseApiUrl.split('/api/'); - apiUrl = `${urlParts[0]}/s/${spaceId}/api/${urlParts[1]}`; - } - - logger.debug(`${FILE_NAME}: Using Elasticsearch API URL: ${apiUrl}`); - - // Add index pattern to rule if provided by space config - if (elasticConfig.space && elasticConfig.space.indexPattern && !rulePayload.index) { - rulePayload.index = Array.isArray(elasticConfig.space.indexPattern) - ? elasticConfig.space.indexPattern - : [elasticConfig.space.indexPattern]; - logger.debug(`${FILE_NAME}: Adding index pattern to rule: ${JSON.stringify(rulePayload.index)}`); - } - - // Send the request to Elasticsearch - const response = await axios({ - method: 'post', - url: apiUrl, - headers: { - 'Content-Type': 'application/json', - 'kbn-xsrf': 'true' - }, - auth: { - username: elasticConfig.username, - password: elasticConfig.password - }, - data: rulePayload - }); - - // Process the response - if (response.status >= 200 && response.status < 300) { - logger.info(`${FILE_NAME}: Successfully sent rule to SIEM in space: ${spaceId}`); - return { - success: true, - status: response.status, - data: response.data, - space: elasticConfig.space - }; - } else { - logger.error(`${FILE_NAME}: Error sending rule to SIEM. Status: ${response.status}, Response: ${JSON.stringify(response.data)}`); - return { - success: false, - status: response.status, - message: `Failed to add rule to SIEM in space ${spaceId}. Status: ${response.status}`, - data: response.data - }; - } - } catch (error) { - logger.error(`${FILE_NAME}: API error sending rule to SIEM: ${error.message}`); - logger.debug(`${FILE_NAME}: API error details: ${error.response ? JSON.stringify(error.response.data) : 'No response data'}`); - - const errorMessage = error.response && error.response.data && error.response.data.message - ? error.response.data.message - : error.message; - - return { - success: false, - message: errorMessage, - error: error - }; - } -}; - /** * Make a generic request to an Elasticsearch API endpoint * @@ -205,7 +123,6 @@ const makeElasticRequest = async (options) => { }; module.exports = { - sendRuleToSiem, makeElasticRequest, getElasticConfig, getAllSpaces diff --git a/src/services/elastic/elastic_send_rule_to_siem_service.js b/src/services/elastic/elastic_send_rule_to_siem_service.js new file mode 100644 index 0000000..aa32d2f --- /dev/null +++ b/src/services/elastic/elastic_send_rule_to_siem_service.js @@ -0,0 +1,96 @@ +/** + * elastic_send_rule_to_siem_service.js + * + * Service for sending rules to Elasticsearch SIEM + */ +const axios = require('axios'); +const logger = require('../../utils/logger'); +const { getElasticConfig } = require('./elastic_api_service'); + +const FILE_NAME = 'elastic_send_rule_to_siem_service.js'; + +/** + * Send a rule to Elasticsearch SIEM in a specific space + * + * @param {Object} rulePayload - The rule payload to send to Elasticsearch + * @param {string} spaceId - The ID of the space to send the rule to + * @returns {Promise} - Object containing success status and response/error information + */ +const sendRuleToSiem = async (rulePayload, spaceId = 'default') => { + logger.info(`${FILE_NAME}: Sending rule to Elasticsearch SIEM in space: ${spaceId}`); + + try { + const elasticConfig = getElasticConfig(spaceId); + const baseApiUrl = elasticConfig.apiEndpoint; + + // Construct space-specific URL if needed + let apiUrl = baseApiUrl; + if (spaceId && spaceId !== 'default') { + // Insert space ID into URL: http://localhost:5601/api/detection_engine/rules + // becomes http://localhost:5601/s/space-id/api/detection_engine/rules + const urlParts = baseApiUrl.split('/api/'); + apiUrl = `${urlParts[0]}/s/${spaceId}/api/${urlParts[1]}`; + } + + logger.debug(`${FILE_NAME}: Using Elasticsearch API URL: ${apiUrl}`); + + // Add index pattern to rule if provided by space config + if (elasticConfig.space && elasticConfig.space.indexPattern && !rulePayload.index) { + rulePayload.index = Array.isArray(elasticConfig.space.indexPattern) + ? elasticConfig.space.indexPattern + : [elasticConfig.space.indexPattern]; + logger.debug(`${FILE_NAME}: Adding index pattern to rule: ${JSON.stringify(rulePayload.index)}`); + } + + // Send the request to Elasticsearch + const response = await axios({ + method: 'post', + url: apiUrl, + headers: { + 'Content-Type': 'application/json', + 'kbn-xsrf': 'true' + }, + auth: { + username: elasticConfig.username, + password: elasticConfig.password + }, + data: rulePayload + }); + + // Process the response + if (response.status >= 200 && response.status < 300) { + logger.info(`${FILE_NAME}: Successfully sent rule to SIEM in space: ${spaceId}`); + return { + success: true, + status: response.status, + data: response.data, + space: elasticConfig.space + }; + } else { + logger.error(`${FILE_NAME}: Error sending rule to SIEM. Status: ${response.status}, Response: ${JSON.stringify(response.data)}`); + return { + success: false, + status: response.status, + message: `Failed to add rule to SIEM in space ${spaceId}. Status: ${response.status}`, + data: response.data + }; + } + } catch (error) { + logger.error(`${FILE_NAME}: API error sending rule to SIEM: ${error.message}`); + logger.debug(`${FILE_NAME}: API error details: ${error.response ? JSON.stringify(error.response.data) : 'No response data'}`); + + const errorMessage = error.response && error.response.data && error.response.data.message + ? error.response.data.message + : error.message; + + return { + success: false, + message: errorMessage, + error: error + }; + } +}; + +module.exports = { + sendRuleToSiem +}; \ No newline at end of file -- 2.39.5 From 31d6296c6e09caa32b962558beac596141b78bae Mon Sep 17 00:00:00 2001 From: Charlotte Croce Date: Fri, 18 Apr 2025 13:26:20 -0400 Subject: [PATCH 03/15] refactor sigma actions handler into multiple files --- slack.example.yml | 4 + src/app.js | 7 +- .../sigma/actions/sigma_action_core.js | 173 ++++ .../sigma/actions/sigma_action_registry.js | 38 + .../sigma/actions/sigma_conversion_actions.js | 58 ++ .../sigma/actions/sigma_siem_actions.js | 357 ++++++++ .../sigma/actions/sigma_view_actions.js | 216 +++++ src/handlers/sigma/sigma_action_handlers.js | 760 ------------------ src/handlers/sigma/sigma_create_handler.js | 4 +- src/handlers/sigma/sigma_details_handler.js | 5 +- 10 files changed, 853 insertions(+), 769 deletions(-) create mode 100644 src/handlers/sigma/actions/sigma_action_core.js create mode 100644 src/handlers/sigma/actions/sigma_action_registry.js create mode 100644 src/handlers/sigma/actions/sigma_conversion_actions.js create mode 100644 src/handlers/sigma/actions/sigma_siem_actions.js create mode 100644 src/handlers/sigma/actions/sigma_view_actions.js delete mode 100644 src/handlers/sigma/sigma_action_handlers.js diff --git a/slack.example.yml b/slack.example.yml index 79ed59b..68f6575 100644 --- a/slack.example.yml +++ b/slack.example.yml @@ -59,6 +59,10 @@ features: url: http://SERVER_DOMAIN_NAME/slack/events description: Show statistics should_escape: false + - command: /fylgja + url: http://SERVER_DOMAIN_NAME/slack/events + description: Run fylgja commands + should_escape: false oauth_config: scopes: bot: diff --git a/src/app.js b/src/app.js index c5e99be..f2cc711 100644 --- a/src/app.js +++ b/src/app.js @@ -17,7 +17,8 @@ const FILE_NAME = getFileName(__filename); const sigmaDetailsHandler = require('./handlers/sigma/sigma_details_handler'); const sigmaSearchHandler = require('./handlers/sigma/sigma_search_handler'); const sigmaCreateHandler = require('./handlers/sigma/sigma_create_handler'); -const sigmaActionHandlers = require('./handlers/sigma/sigma_action_handlers'); +// Import the action registry +const sigmaActionRegistry = require('./handlers/sigma/actions/sigma_action_registry'); //const configCommand = require('./commands/config/index.js'); //const alertsCommand = require('./commands/alerts/index.js'); //const caseCommand = require('./commands/case/index.js'); @@ -113,8 +114,8 @@ app.command('/sigma-stats', async ({ command, ack, respond }) => { } }); -// Register all button action handlers from centralized module -sigmaActionHandlers.registerActionHandlers(app); +// Register all button action handlers from the modular registry +sigmaActionRegistry.registerActionHandlers(app); /** * Listen for any message in DMs diff --git a/src/handlers/sigma/actions/sigma_action_core.js b/src/handlers/sigma/actions/sigma_action_core.js new file mode 100644 index 0000000..7a9af17 --- /dev/null +++ b/src/handlers/sigma/actions/sigma_action_core.js @@ -0,0 +1,173 @@ +/** + * sigma_action_core.js + * + * Core utility functions for Sigma-related Slack actions + */ +const logger = require('../../../utils/logger'); +const { handleError } = require('../../../utils/error_handler'); +const { explainSigmaRule } = require('../../../services/sigma/sigma_details_service'); +const { convertRuleToBackend } = require('../../../services/sigma/sigma_backend_converter'); +const { getRuleExplanationBlocks } = require('../../../blocks/sigma/sigma_details_block'); +const { getConversionResultBlocks } = require('../../../blocks/sigma/sigma_conversion_block'); + +const { SIGMA_CLI_CONFIG } = require('../../../config/appConfig'); + +const FILE_NAME = 'sigma_action_core.js'; + +/** + * Process and display details for a Sigma rule + * + * @param {string} ruleId - The ID of the rule to get details for + * @param {Function} respond - Function to send response back to Slack + * @param {boolean} replaceOriginal - Whether to replace the original message + * @param {string} responseType - Response type (ephemeral or in_channel) + * @returns {Promise} + */ +const processRuleDetails = async (ruleId, respond, replaceOriginal = false, responseType = 'in_channel') => { + try { + if (!ruleId) { + logger.warn(`${FILE_NAME}: Missing rule ID in processRuleDetails`); + await respond({ + text: 'Error: Missing rule ID for details', + replace_original: replaceOriginal, + response_type: responseType + }); + return; + } + + logger.info(`${FILE_NAME}: Processing details for sigma rule: ${ruleId}`); + + // Get Sigma rule details + logger.info(`${FILE_NAME}: Calling explainSigmaRule with ID: '${ruleId}'`); + const result = await explainSigmaRule(ruleId); + + if (!result.success) { + logger.error(`${FILE_NAME}: Rule details retrieval failed: ${result.message}`); + await respond({ + text: `Error: ${result.message}`, + replace_original: replaceOriginal, + response_type: responseType + }); + return; + } + + if (!result.explanation) { + logger.error(`${FILE_NAME}: Rule details succeeded but no explanation object was returned`); + await respond({ + text: 'Error: Generated details were empty', + replace_original: replaceOriginal, + response_type: responseType + }); + return; + } + + logger.info(`${FILE_NAME}: Rule ${ruleId} details retrieved successfully`); + + // Generate blocks + let blocks; + try { + blocks = getRuleExplanationBlocks(result.explanation); + } catch (blockError) { + await handleError(blockError, `${FILE_NAME}: Block generation`, respond, { + replaceOriginal: replaceOriginal, + responseType: responseType, + customMessage: `Rule ${result.explanation.id}: ${result.explanation.title}\n${result.explanation.description}` + }); + return; + } + + // Respond with the details + await respond({ + blocks: blocks, + replace_original: replaceOriginal, + response_type: responseType + }); + } catch (error) { + await handleError(error, `${FILE_NAME}: Process rule details`, respond, { + replaceOriginal: replaceOriginal, + responseType: responseType + }); + } +}; + +/** + * Process and convert a Sigma rule to the target backend format + * + * @param {string} ruleId - The ID of the rule to convert + * @param {Object} config - Configuration for the conversion (backend, target, format) + * @param {Function} respond - Function to send response back to Slack + * @param {boolean} replaceOriginal - Whether to replace the original message + * @param {string} responseType - Response type (ephemeral or in_channel) + * @returns {Promise} + */ +const processRuleConversion = async (ruleId, config, respond, replaceOriginal = false, responseType = 'in_channel') => { + try { + if (!ruleId) { + logger.warn(`${FILE_NAME}: Missing rule ID in processRuleConversion`); + await respond({ + text: 'Error: Missing rule ID for conversion', + replace_original: replaceOriginal, + response_type: responseType + }); + return; + } + + logger.info(`${FILE_NAME}: Processing conversion for sigma rule: ${ruleId}`); + + // Set default configuration from YAML config if not provided + const conversionConfig = config || { + backend: SIGMA_CLI_CONFIG.backend, + target: SIGMA_CLI_CONFIG.target, + format: SIGMA_CLI_CONFIG.format + }; + + await respond({ + text: `Converting rule ${ruleId} using ${conversionConfig.backend}/${conversionConfig.target} to ${conversionConfig.format}...`, + replace_original: replaceOriginal, + response_type: 'ephemeral' + }); + + // Get the rule and convert it + const conversionResult = await convertRuleToBackend(ruleId, conversionConfig); + + if (!conversionResult.success) { + logger.error(`${FILE_NAME}: Rule conversion failed: ${conversionResult.message}`); + await respond({ + text: `Error: ${conversionResult.message}`, + replace_original: replaceOriginal, + response_type: responseType + }); + return; + } + + // Generate blocks for displaying the result + let blocks; + try { + blocks = getConversionResultBlocks(conversionResult); + } catch (blockError) { + await handleError(blockError, `${FILE_NAME}: Block generation`, respond, { + replaceOriginal: replaceOriginal, + responseType: responseType, + customMessage: `Rule ${ruleId} converted successfully. Use the following output with your SIEM:\n\`\`\`\n${conversionResult.output}\n\`\`\`` + }); + return; + } + + // Respond with the conversion result + await respond({ + blocks: blocks, + replace_original: replaceOriginal, + response_type: responseType + }); + } catch (error) { + await handleError(error, `${FILE_NAME}: Process rule conversion`, respond, { + replaceOriginal: replaceOriginal, + responseType: responseType + }); + } +}; + +module.exports = { + processRuleDetails, + processRuleConversion +}; \ No newline at end of file diff --git a/src/handlers/sigma/actions/sigma_action_registry.js b/src/handlers/sigma/actions/sigma_action_registry.js new file mode 100644 index 0000000..9f78726 --- /dev/null +++ b/src/handlers/sigma/actions/sigma_action_registry.js @@ -0,0 +1,38 @@ +/** + * sigma_action_registry.js + * + * Main registry that imports and registers all Sigma action handlers + */ +const logger = require('../../../utils/logger'); +const { registerViewActions } = require('./sigma_view_actions'); +const { registerConversionActions } = require('./sigma_conversion_actions'); +const { registerSiemActions } = require('./sigma_siem_actions'); +const { processRuleDetails, processRuleConversion } = require('./sigma_action_core'); + +const FILE_NAME = 'sigma_action_registry.js'; + +/** + * Register all Sigma-related action handlers + * + * @param {Object} app - The Slack app instance + */ +const registerActionHandlers = (app) => { + logger.info(`${FILE_NAME}: Registering all sigma action handlers`); + + // Register view-related handlers (view YAML, view details, pagination) + registerViewActions(app); + + // Register conversion-related handlers + registerConversionActions(app); + + // Register SIEM-related handlers (send to SIEM, space selection) + registerSiemActions(app); + + logger.info(`${FILE_NAME}: All sigma action handlers registered successfully`); +}; + +module.exports = { + registerActionHandlers, + processRuleDetails, + processRuleConversion +}; \ No newline at end of file diff --git a/src/handlers/sigma/actions/sigma_conversion_actions.js b/src/handlers/sigma/actions/sigma_conversion_actions.js new file mode 100644 index 0000000..04ca34f --- /dev/null +++ b/src/handlers/sigma/actions/sigma_conversion_actions.js @@ -0,0 +1,58 @@ +/** + * sigma_conversion_actions.js + * + * Handlers for Sigma rule conversion actions + */ +const logger = require('../../../utils/logger'); +const { handleError } = require('../../../utils/error_handler'); +const { processRuleConversion } = require('./sigma_action_core'); + +const FILE_NAME = 'sigma_conversion_actions.js'; + +/** + * Register conversion-related action handlers + * + * @param {Object} app - The Slack app instance + */ +const registerConversionActions = (app) => { + logger.info(`${FILE_NAME}: Registering conversion-related action handlers`); + + // Handle convert_rule_to_siem button clicks + app.action('convert_rule_to_siem', async ({ body, ack, respond }) => { + try { + await ack(); + logger.debug(`${FILE_NAME}: convert_rule_to_siem action received: ${JSON.stringify(body.actions)}`); + + if (!body || !body.actions || !body.actions[0] || !body.actions[0].value) { + logger.error(`${FILE_NAME}: Invalid action payload: missing rule ID`); + await respond({ + text: 'Error: Could not determine which rule to convert', + replace_original: false + }); + return; + } + + // Extract rule ID from button value + const ruleId = body.actions[0].value.replace('convert_rule_to_siem_', ''); + logger.info(`${FILE_NAME}: convert_rule_to_siem button clicked for rule: ${ruleId}`); + + const config = { + backend: 'lucene', + target: 'ecs_windows', + format: 'siem_rule_ndjson' + }; + + await processRuleConversion(ruleId, config, respond, false, 'in_channel'); + } catch (error) { + await handleError(error, `${FILE_NAME}: convert_rule_to_siem action`, respond, { + replaceOriginal: false + }); + } + }); + + logger.info(`${FILE_NAME}: All conversion action handlers registered successfully`); +}; + +module.exports = { + registerConversionActions +}; \ No newline at end of file diff --git a/src/handlers/sigma/actions/sigma_siem_actions.js b/src/handlers/sigma/actions/sigma_siem_actions.js new file mode 100644 index 0000000..d446fcd --- /dev/null +++ b/src/handlers/sigma/actions/sigma_siem_actions.js @@ -0,0 +1,357 @@ +/** + * sigma_siem_actions.js + * + * Handlers for sending Sigma rules to SIEM and space-related operations + */ +const logger = require('../../../utils/logger'); +const { handleError } = require('../../../utils/error_handler'); +const { explainSigmaRule } = require('../../../services/sigma/sigma_details_service'); +const { convertRuleToBackend } = require('../../../services/sigma/sigma_backend_converter'); +const { sendRuleToSiem } = require('../../../services/elastic/elastic_send_rule_to_siem_service'); +const { getAllSpaces } = require('../../../services/elastic/elastic_api_service'); +const { getSpaceSelectionBlocks } = require('../../../blocks/sigma/sigma_space_selection_block'); + +const { SIGMA_CLI_CONFIG } = require('../../../config/appConfig'); + +const FILE_NAME = 'sigma_siem_actions.js'; + +/** + * Parse JSON rule payload and add required fields + * + * @param {string} ruleOutput - The JSON rule as string + * @param {string} ruleId - The rule ID + * @param {Object} conversionResult - Result from convertRuleToBackend + * @param {Object} selectedSpace - Optional space configuration + * @returns {Object} Prepared rule payload + * @throws {Error} If JSON parsing fails + */ +const prepareRulePayload = (ruleOutput, ruleId, conversionResult, selectedSpace = null) => { + const rulePayload = JSON.parse(ruleOutput); + + // Add required fields if not present + rulePayload.rule_id = rulePayload.rule_id || ruleId; + rulePayload.from = rulePayload.from || "now-360s"; + rulePayload.to = rulePayload.to || "now"; + rulePayload.interval = rulePayload.interval || "5m"; + + // Set index pattern from space configuration if available + if (selectedSpace && selectedSpace.indexPattern) { + rulePayload.index = Array.isArray(selectedSpace.indexPattern) + ? selectedSpace.indexPattern + : [selectedSpace.indexPattern]; + logger.debug(`${FILE_NAME}: Setting index pattern from space config: ${JSON.stringify(rulePayload.index)}`); + } + + // Make sure required fields are present + if (!rulePayload.name) { + rulePayload.name = conversionResult.rule?.title || `Sigma Rule ${ruleId}`; + } + + if (!rulePayload.description) { + rulePayload.description = conversionResult.rule?.description || + `Converted from Sigma rule: ${ruleId}`; + } + + if (!rulePayload.risk_score) { + // Map Sigma level to risk score + const levelMap = { + 'critical': 90, + 'high': 73, + 'medium': 50, + 'low': 25, + 'informational': 10 + }; + + rulePayload.risk_score = levelMap[conversionResult.rule?.level] || 50; + } + + if (!rulePayload.severity) { + rulePayload.severity = conversionResult.rule?.level || 'medium'; + } + + if (!rulePayload.enabled) { + rulePayload.enabled = true; + } + + return rulePayload; +}; + +/** + * Register SIEM and space-related action handlers + * + * @param {Object} app - The Slack app instance + */ +const registerSiemActions = (app) => { + logger.info(`${FILE_NAME}: Registering SIEM-related action handlers`); + + // Handle "Send to SIEM" button clicks + app.action('send_sigma_rule_to_siem', async ({ body, ack, respond }) => { + try { + await ack(); + logger.debug(`${FILE_NAME}: send_sigma_rule_to_siem action received: ${JSON.stringify(body.actions)}`); + + if (!body || !body.actions || !body.actions[0] || !body.actions[0].value) { + logger.error(`${FILE_NAME}: Invalid action payload: missing rule ID`); + await respond({ + text: 'Error: Could not determine which rule to send', + replace_original: false, + response_type: 'ephemeral' + }); + return; + } + + // Extract rule ID from action value + // Value format is "send_sigma_rule_to_siem_[ruleID]" + const actionValue = body.actions[0].value; + const ruleId = actionValue.replace('send_sigma_rule_to_siem_', ''); + + if (!ruleId) { + logger.error(`${FILE_NAME}: Missing rule ID in action value: ${actionValue}`); + await respond({ + text: 'Error: Missing rule ID in button data', + replace_original: false, + response_type: 'ephemeral' + }); + return; + } + + logger.info(`${FILE_NAME}: Sending rule ${ruleId} to SIEM`); + + // Inform user that processing is happening + await respond({ + text: `Sending rule ${ruleId} to Elasticsearch SIEM...`, + replace_original: false, + response_type: 'ephemeral' + }); + + // Get the converted rule in Elasticsearch format using config from YAML + const config = { + backend: SIGMA_CLI_CONFIG.backend, + target: SIGMA_CLI_CONFIG.target, + format: SIGMA_CLI_CONFIG.format + }; + + logger.info(`${FILE_NAME}: Converting rule ${ruleId} for SIEM export`); + const conversionResult = await convertRuleToBackend(ruleId, config); + + if (!conversionResult.success) { + logger.error(`${FILE_NAME}: Rule conversion failed: ${conversionResult.message}`); + await respond({ + text: `Error: Failed to convert rule for SIEM: ${conversionResult.message}`, + replace_original: false, + response_type: 'ephemeral' + }); + return; + } + + // Parse the converted rule JSON + let rulePayload; + try { + rulePayload = prepareRulePayload(conversionResult.output, ruleId, conversionResult); + } catch (parseError) { + logger.error(`${FILE_NAME}: Failed to parse converted rule JSON: ${parseError.message}`); + await respond({ + text: `Error: The converted rule is not valid JSON: ${parseError.message}`, + replace_original: false, + response_type: 'ephemeral' + }); + return; + } + + // Send the rule to Elasticsearch using api service + try { + const result = await sendRuleToSiem(rulePayload); + + if (result.success) { + logger.info(`${FILE_NAME}: Successfully sent rule ${ruleId} to SIEM`); + await respond({ + text: `✅ Success! Rule "${rulePayload.name}" has been added to your Elasticsearch SIEM.`, + replace_original: false, + response_type: 'in_channel' + }); + } else { + logger.error(`${FILE_NAME}: Error sending rule to SIEM: ${result.message}`); + await respond({ + text: `Error: Failed to add rule to SIEM: ${result.message}`, + replace_original: false, + response_type: 'ephemeral' + }); + } + } catch (error) { + await handleError(error, `${FILE_NAME}: send_sigma_rule_to_siem action`, respond, { + replaceOriginal: false + }); + } + } catch (error) { + await handleError(error, `${FILE_NAME}: send_sigma_rule_to_siem action`, respond, { + replaceOriginal: false + }); + } + }); + + // Handle space selection button click + app.action('select_space_for_rule', async ({ body, ack, respond }) => { + try { + await ack(); + logger.debug(`${FILE_NAME}: select_space_for_rule action received: ${JSON.stringify(body.actions)}`); + + if (!body || !body.actions || !body.actions[0] || !body.actions[0].value) { + logger.error(`${FILE_NAME}: Invalid action payload: missing rule ID`); + await respond({ + text: 'Error: Could not determine which rule to select space for', + replace_original: false, + response_type: 'ephemeral' + }); + return; + } + + // Extract rule ID from value + const actionValue = body.actions[0].value; + const ruleId = actionValue.replace('select_space_for_rule_', ''); + + // Get rule information to display in the space selection message + const explainResult = await explainSigmaRule(ruleId); + const ruleInfo = explainResult.success ? explainResult.explanation : { title: ruleId }; + + // Generate blocks for space selection + const blocks = getSpaceSelectionBlocks(ruleId, ruleInfo); + + // Show space selection options + await respond({ + blocks: blocks, + replace_original: false, + response_type: 'ephemeral' + }); + + } catch (error) { + await handleError(error, `${FILE_NAME}: select_space_for_rule action`, respond, { + replaceOriginal: false + }); + } + }); + + // Handle space selection cancel button + app.action('cancel_space_selection', async ({ body, ack, respond }) => { + try { + await ack(); + await respond({ + text: 'Space selection cancelled.', + replace_original: false, + response_type: 'ephemeral' + }); + } catch (error) { + await handleError(error, `${FILE_NAME}: cancel_space_selection action`, respond, { + replaceOriginal: false + }); + } + }); + + // Dynamic handler for all space selection buttons + // This uses a pattern matcher to match any action ID that starts with "send_rule_to_space_" + app.action(/^send_rule_to_space_(.*)$/, async ({ body, action, ack, respond }) => { + try { + await ack(); + logger.debug(`${FILE_NAME}: Space selection action received: ${JSON.stringify(action)}`); + + // Extract rule ID and space ID from the action value + const actionValue = action.value; + const parts = actionValue.split('_'); + const spaceId = parts.pop(); // Last part is the space ID + const ruleId = actionValue.match(/send_rule_to_space_(.+)_/)[1]; // Extract full UUID + + logger.info(`${FILE_NAME}: Selected space ${spaceId} for rule ${ruleId}`); + + + // Get space info + const spaces = getAllSpaces(); + const selectedSpace = spaces.find(s => s.id === spaceId); + + if (!selectedSpace) { + logger.error(`${FILE_NAME}: Space not found: ${spaceId}`); + await respond({ + text: `Error: Space "${spaceId}" not found in configuration`, + replace_original: false, + response_type: 'ephemeral' + }); + return; + } + + // Inform user that processing is happening + await respond({ + text: `Sending rule ${ruleId} to ${selectedSpace.emoji || ''} ${selectedSpace.name} space...`, + replace_original: false, + response_type: 'ephemeral' + }); + + // Get the converted rule in Elasticsearch format + const config = { + backend: SIGMA_CLI_CONFIG.backend, + target: SIGMA_CLI_CONFIG.target, + format: SIGMA_CLI_CONFIG.format + }; + + logger.info(`${FILE_NAME}: Converting rule ${ruleId} for SIEM export to space ${spaceId}`); + const conversionResult = await convertRuleToBackend(ruleId, config); + + if (!conversionResult.success) { + logger.error(`${FILE_NAME}: Rule conversion failed: ${conversionResult.message}`); + await respond({ + text: `Error: Failed to convert rule for SIEM: ${conversionResult.message}`, + replace_original: false, + response_type: 'ephemeral' + }); + return; + } + + // Parse the converted rule JSON + let rulePayload; + try { + rulePayload = prepareRulePayload(conversionResult.output, ruleId, conversionResult, selectedSpace); + } catch (parseError) { + logger.error(`${FILE_NAME}: Failed to parse converted rule JSON: ${parseError.message}`); + await respond({ + text: `Error: The converted rule is not valid JSON: ${parseError.message}`, + replace_original: false, + response_type: 'ephemeral' + }); + return; + } + + // Send the rule to the selected Elasticsearch space + try { + const result = await sendRuleToSiem(rulePayload, spaceId); + + if (result.success) { + logger.info(`${FILE_NAME}: Successfully sent rule ${ruleId} to space ${spaceId}`); + await respond({ + text: `✅ Success! Rule "${rulePayload.name}" has been added to the ${selectedSpace.emoji || ''} ${selectedSpace.name} space in Elasticsearch.`, + replace_original: false, + response_type: 'in_channel' + }); + } else { + logger.error(`${FILE_NAME}: Error sending rule to SIEM: ${result.message}`); + await respond({ + text: `Error: Failed to add rule to the ${selectedSpace.name} space: ${result.message}`, + replace_original: false, + response_type: 'ephemeral' + }); + } + } catch (error) { + await handleError(error, `${FILE_NAME}: send_rule_to_space action`, respond, { + replaceOriginal: false + }); + } + } catch (error) { + await handleError(error, `${FILE_NAME}: send_rule_to_space action`, respond, { + replaceOriginal: false + }); + } + }); + + logger.info(`${FILE_NAME}: All SIEM action handlers registered successfully`); +}; + +module.exports = { + registerSiemActions, + prepareRulePayload +}; \ No newline at end of file diff --git a/src/handlers/sigma/actions/sigma_view_actions.js b/src/handlers/sigma/actions/sigma_view_actions.js new file mode 100644 index 0000000..d896f31 --- /dev/null +++ b/src/handlers/sigma/actions/sigma_view_actions.js @@ -0,0 +1,216 @@ +/** + * sigma_view_actions.js + * + * Handlers for viewing Sigma rule data and search results + */ +const logger = require('../../../utils/logger'); +const { handleError } = require('../../../utils/error_handler'); +const { getSigmaRuleYaml } = require('../../../services/sigma/sigma_details_service'); +const { searchSigmaRules } = require('../../../services/sigma/sigma_search_service'); +const { getYamlViewBlocks } = require('../../../blocks/sigma/sigma_view_yaml_block'); +const { getSearchResultBlocks } = require('../../../blocks/sigma/sigma_search_results_block'); +const { processRuleDetails } = require('./sigma_action_core'); + +const FILE_NAME = 'sigma_view_actions.js'; + +/** + * Handle pagination actions (Previous, Next) + * + * @param {Object} body - The action payload body + * @param {Function} ack - Function to acknowledge the action + * @param {Function} respond - Function to send response + */ +const handlePaginationAction = async (body, ack, respond) => { + try { + await ack(); + logger.debug(`${FILE_NAME}: Pagination action received: ${JSON.stringify(body.actions)}`); + + if (!body || !body.actions || !body.actions[0] || !body.actions[0].value) { + logger.error(`${FILE_NAME}: Invalid pagination action payload: missing parameters`); + await respond({ + text: 'Error: Could not process pagination request', + replace_original: false + }); + return; + } + + // Parse the action value which contains our pagination parameters + const action = body.actions[0]; + let valueData; + + try { + valueData = JSON.parse(action.value); + } catch (parseError) { + await handleError(parseError, `${FILE_NAME}: Pagination value parsing`, respond, { + replaceOriginal: false, + customMessage: 'Error: Invalid pagination parameters' + }); + return; + } + + const { keyword, page, pageSize } = valueData; + + if (!keyword) { + logger.warn(`${FILE_NAME}: Missing keyword in pagination action`); + await respond({ + text: 'Error: Missing search keyword in pagination request', + replace_original: false + }); + return; + } + + logger.info(`${FILE_NAME}: Processing pagination request for "${keyword}" (page ${page}, size ${pageSize})`); + + // Perform the search with the new pagination parameters + const searchResult = await searchSigmaRules(keyword, page, pageSize); + + if (!searchResult.success) { + logger.error(`${FILE_NAME}: Search failed during pagination: ${searchResult.message}`); + await respond({ + text: `Error: ${searchResult.message}`, + replace_original: false + }); + return; + } + + // Generate the updated blocks for the search results + let blocks; + try { + blocks = getSearchResultBlocks( + keyword, + searchResult.results, + searchResult.pagination + ); + } catch (blockError) { + await handleError(blockError, `${FILE_NAME}: Pagination block generation`, respond, { + replaceOriginal: false, + customMessage: `Error generating results view: ${blockError.message}` + }); + return; + } + + // Return the response that will update the original message + await respond({ + blocks: blocks, + replace_original: true + }); + } catch (error) { + await handleError(error, `${FILE_NAME}: Pagination action handler`, respond, { + replaceOriginal: false + }); + } +}; + +/** + * Register view-related action handlers + * + * @param {Object} app - The Slack app instance + */ +const registerViewActions = (app) => { + logger.info(`${FILE_NAME}: Registering view-related action handlers`); + + // Handle View YAML button clicks + app.action('view_yaml', async ({ body, ack, respond }) => { + logger.info(`${FILE_NAME}: VIEW_YAML ACTION TRIGGERED`); + try { + await ack(); + logger.debug(`${FILE_NAME}: View YAML action received: ${JSON.stringify(body.actions)}`); + + if (!body || !body.actions || !body.actions[0] || !body.actions[0].value) { + logger.error(`${FILE_NAME}: Invalid action payload: missing rule ID`); + await respond({ + text: 'Error: Could not determine which rule to get YAML for', + replace_original: false + }); + return; + } + + // Extract rule ID from button value + // Handle both formats: direct ID from search results or view_yaml_{ruleId} from details view + let ruleId = body.actions[0].value; + if (ruleId.startsWith('view_yaml_')) { + ruleId = ruleId.replace('view_yaml_', ''); + } + + logger.info(`${FILE_NAME}: View YAML button clicked for rule: ${ruleId}`); + + // Get Sigma rule YAML + const result = await getSigmaRuleYaml(ruleId); + logger.debug(`${FILE_NAME}: YAML retrieval result: ${JSON.stringify(result, null, 2)}`); + + if (!result.success) { + logger.error(`${FILE_NAME}: Rule YAML retrieval failed: ${result.message}`); + await respond({ + text: `Error: ${result.message}`, + replace_original: false + }); + return; + } + + logger.info(`${FILE_NAME}: Rule ${ruleId} YAML retrieved successfully via button click`); + + // Use the module to generate blocks + const blocks = getYamlViewBlocks(ruleId, result.yaml || ''); + + // Respond with the YAML content + await respond({ + blocks: blocks, + replace_original: false + }); + } catch (error) { + await handleError(error, `${FILE_NAME}: View YAML action`, respond, { + replaceOriginal: false + }); + } + }); + + // Handle "View Rule Details" button clicks from search results + app.action('view_rule_details', async ({ body, ack, respond }) => { + logger.info(`${FILE_NAME}: VIEW_RULE_DETAILS ACTION TRIGGERED`); + try { + await ack(); + logger.debug(`${FILE_NAME}: View Rule Details action received: ${JSON.stringify(body.actions)}`); + + if (!body || !body.actions || !body.actions[0] || !body.actions[0].value) { + logger.error(`${FILE_NAME}: Invalid action payload: missing rule ID`); + await respond({ + text: 'Error: Could not determine which rule to explain', + replace_original: false + }); + return; + } + + const ruleId = body.actions[0].value; + logger.info(`${FILE_NAME}: Rule details button clicked for rule ID: ${ruleId}`); + + // Inform user we're processing + await respond({ + text: `Processing details for rule ${ruleId}...`, + replace_original: false, + response_type: 'ephemeral' + }); + + await processRuleDetails(ruleId, respond, false, 'in_channel'); + } catch (error) { + await handleError(error, `${FILE_NAME}: View rule details action`, respond, { + replaceOriginal: false + }); + } + }); + + // Handle pagination button clicks + app.action('search_prev_page', async ({ body, ack, respond }) => { + await handlePaginationAction(body, ack, respond); + }); + + app.action('search_next_page', async ({ body, ack, respond }) => { + await handlePaginationAction(body, ack, respond); + }); + + logger.info(`${FILE_NAME}: All view action handlers registered successfully`); +}; + +module.exports = { + registerViewActions, + handlePaginationAction +}; \ No newline at end of file diff --git a/src/handlers/sigma/sigma_action_handlers.js b/src/handlers/sigma/sigma_action_handlers.js deleted file mode 100644 index 81a0445..0000000 --- a/src/handlers/sigma/sigma_action_handlers.js +++ /dev/null @@ -1,760 +0,0 @@ -/** - * sigma_action_handlers.js - * - * Centralized action handlers for Sigma-related Slack interactions - */ -const logger = require('../../utils/logger'); -const { handleError } = require('../../utils/error_handler'); -const { explainSigmaRule, getSigmaRuleYaml } = require('../../services/sigma/sigma_details_service'); -const { convertRuleToBackend } = require('../../services/sigma/sigma_backend_converter'); -const { searchSigmaRules } = require('../../services/sigma/sigma_search_service'); -const { getYamlViewBlocks } = require('../../blocks/sigma/sigma_view_yaml_block'); -const { getSearchResultBlocks } = require('../../blocks/sigma/sigma_search_results_block'); -const { getConversionResultBlocks } = require('../../blocks/sigma/sigma_conversion_block'); -const { getRuleExplanationBlocks } = require('../../blocks/sigma/sigma_details_block'); -const { sendRuleToSiem } = require('../../services/elastic/elastic_send_rule_to_siem_service'); -const { getAllSpaces } = require('../../services/elastic/elastic_api_service'); -const { getSpaceSelectionBlocks } = require('../../blocks/sigma/sigma_space_selection_block'); - -const { SIGMA_CLI_CONFIG, ELASTICSEARCH_CONFIG } = require('../../config/appConfig'); - -const FILE_NAME = 'sigma_action_handlers.js'; - -/** - * Process and display details for a Sigma rule - * - * @param {string} ruleId - The ID of the rule to get details for - * @param {Function} respond - Function to send response back to Slack - * @param {boolean} replaceOriginal - Whether to replace the original message - * @param {string} responseType - Response type (ephemeral or in_channel) - * @returns {Promise} - */ -const processRuleDetails = async (ruleId, respond, replaceOriginal = false, responseType = 'in_channel') => { - try { - if (!ruleId) { - logger.warn(`${FILE_NAME}: Missing rule ID in processRuleDetails`); - await respond({ - text: 'Error: Missing rule ID for details', - replace_original: replaceOriginal, - response_type: responseType - }); - return; - } - - logger.info(`${FILE_NAME}: Processing details for sigma rule: ${ruleId}`); - - // Get Sigma rule details - logger.info(`${FILE_NAME}: Calling explainSigmaRule with ID: '${ruleId}'`); - const result = await explainSigmaRule(ruleId); - - if (!result.success) { - logger.error(`${FILE_NAME}: Rule details retrieval failed: ${result.message}`); - await respond({ - text: `Error: ${result.message}`, - replace_original: replaceOriginal, - response_type: responseType - }); - return; - } - - if (!result.explanation) { - logger.error(`${FILE_NAME}: Rule details succeeded but no explanation object was returned`); - await respond({ - text: 'Error: Generated details were empty', - replace_original: replaceOriginal, - response_type: responseType - }); - return; - } - - logger.info(`${FILE_NAME}: Rule ${ruleId} details retrieved successfully`); - - // Generate blocks - let blocks; - try { - blocks = getRuleExplanationBlocks(result.explanation); - } catch (blockError) { - await handleError(blockError, `${FILE_NAME}: Block generation`, respond, { - replaceOriginal: replaceOriginal, - responseType: responseType, - customMessage: `Rule ${result.explanation.id}: ${result.explanation.title}\n${result.explanation.description}` - }); - return; - } - - // Respond with the details - await respond({ - blocks: blocks, - replace_original: replaceOriginal, - response_type: responseType - }); - } catch (error) { - await handleError(error, `${FILE_NAME}: Process rule details`, respond, { - replaceOriginal: replaceOriginal, - responseType: responseType - }); - } -}; - -/** - * Process and convert a Sigma rule to the target backend format - * - * @param {string} ruleId - The ID of the rule to convert - * @param {Object} config - Configuration for the conversion (backend, target, format) - * @param {Function} respond - Function to send response back to Slack - * @param {boolean} replaceOriginal - Whether to replace the original message - * @param {string} responseType - Response type (ephemeral or in_channel) - * @returns {Promise} - */ -const processRuleConversion = async (ruleId, config, respond, replaceOriginal = false, responseType = 'in_channel') => { - try { - if (!ruleId) { - logger.warn(`${FILE_NAME}: Missing rule ID in processRuleConversion`); - await respond({ - text: 'Error: Missing rule ID for conversion', - replace_original: replaceOriginal, - response_type: responseType - }); - return; - } - - logger.info(`${FILE_NAME}: Processing conversion for sigma rule: ${ruleId}`); - - // Set default configuration from YAML config if not provided - const conversionConfig = config || { - backend: SIGMA_CLI_CONFIG.backend, - target: SIGMA_CLI_CONFIG.target, - format: SIGMA_CLI_CONFIG.format - }; - - await respond({ - text: `Converting rule ${ruleId} using ${conversionConfig.backend}/${conversionConfig.target} to ${conversionConfig.format}...`, - replace_original: replaceOriginal, - response_type: 'ephemeral' - }); - - // Get the rule and convert it - const conversionResult = await convertRuleToBackend(ruleId, conversionConfig); - - if (!conversionResult.success) { - logger.error(`${FILE_NAME}: Rule conversion failed: ${conversionResult.message}`); - await respond({ - text: `Error: ${conversionResult.message}`, - replace_original: replaceOriginal, - response_type: responseType - }); - return; - } - - // Generate blocks for displaying the result - let blocks; - try { - blocks = getConversionResultBlocks(conversionResult); - } catch (blockError) { - await handleError(blockError, `${FILE_NAME}: Block generation`, respond, { - replaceOriginal: replaceOriginal, - responseType: responseType, - customMessage: `Rule ${ruleId} converted successfully. Use the following output with your SIEM:\n\`\`\`\n${conversionResult.output}\n\`\`\`` - }); - return; - } - - // Respond with the conversion result - await respond({ - blocks: blocks, - replace_original: replaceOriginal, - response_type: responseType - }); - } catch (error) { - await handleError(error, `${FILE_NAME}: Process rule conversion`, respond, { - replaceOriginal: replaceOriginal, - responseType: responseType - }); - } -}; - -/** - * Handle pagination actions (Previous, Next) - * - * @param {Object} body - The action payload body - * @param {Function} ack - Function to acknowledge the action - * @param {Function} respond - Function to send response - */ -const handlePaginationAction = async (body, ack, respond) => { - try { - await ack(); - logger.debug(`${FILE_NAME}: Pagination action received: ${JSON.stringify(body.actions)}`); - - if (!body || !body.actions || !body.actions[0] || !body.actions[0].value) { - logger.error(`${FILE_NAME}: Invalid pagination action payload: missing parameters`); - await respond({ - text: 'Error: Could not process pagination request', - replace_original: false - }); - return; - } - - // Parse the action value which contains our pagination parameters - const action = body.actions[0]; - let valueData; - - try { - valueData = JSON.parse(action.value); - } catch (parseError) { - await handleError(parseError, `${FILE_NAME}: Pagination value parsing`, respond, { - replaceOriginal: false, - customMessage: 'Error: Invalid pagination parameters' - }); - return; - } - - const { keyword, page, pageSize } = valueData; - - if (!keyword) { - logger.warn(`${FILE_NAME}: Missing keyword in pagination action`); - await respond({ - text: 'Error: Missing search keyword in pagination request', - replace_original: false - }); - return; - } - - logger.info(`${FILE_NAME}: Processing pagination request for "${keyword}" (page ${page}, size ${pageSize})`); - - // Perform the search with the new pagination parameters - const searchResult = await searchSigmaRules(keyword, page, pageSize); - - if (!searchResult.success) { - logger.error(`${FILE_NAME}: Search failed during pagination: ${searchResult.message}`); - await respond({ - text: `Error: ${searchResult.message}`, - replace_original: false - }); - return; - } - - // Generate the updated blocks for the search results - let blocks; - try { - blocks = getSearchResultBlocks( - keyword, - searchResult.results, - searchResult.pagination - ); - } catch (blockError) { - await handleError(blockError, `${FILE_NAME}: Pagination block generation`, respond, { - replaceOriginal: false, - customMessage: `Error generating results view: ${blockError.message}` - }); - return; - } - - // Return the response that will update the original message - await respond({ - blocks: blocks, - replace_original: true - }); - } catch (error) { - await handleError(error, `${FILE_NAME}: Pagination action handler`, respond, { - replaceOriginal: false - }); - } -}; - -/** - * Register all Sigma-related action handlers - * - * @param {Object} app - The Slack app instance - */ -const registerActionHandlers = (app) => { - logger.info(`${FILE_NAME}: Registering consolidated sigma action handlers`); - - // Handle "Send to SIEM" button clicks - app.action('send_sigma_rule_to_siem', async ({ body, ack, respond }) => { - try { - await ack(); - logger.debug(`${FILE_NAME}: send_sigma_rule_to_siem action received: ${JSON.stringify(body.actions)}`); - - if (!body || !body.actions || !body.actions[0] || !body.actions[0].value) { - logger.error(`${FILE_NAME}: Invalid action payload: missing rule ID`); - await respond({ - text: 'Error: Could not determine which rule to send', - replace_original: false, - response_type: 'ephemeral' - }); - return; - } - - // Extract rule ID from action value - // Value format is "send_sigma_rule_to_siem_[ruleID]" - const actionValue = body.actions[0].value; - const ruleId = actionValue.replace('send_sigma_rule_to_siem_', ''); - - if (!ruleId) { - logger.error(`${FILE_NAME}: Missing rule ID in action value: ${actionValue}`); - await respond({ - text: 'Error: Missing rule ID in button data', - replace_original: false, - response_type: 'ephemeral' - }); - return; - } - - logger.info(`${FILE_NAME}: Sending rule ${ruleId} to SIEM`); - - // Inform user that processing is happening - await respond({ - text: `Sending rule ${ruleId} to Elasticsearch SIEM...`, - replace_original: false, - response_type: 'ephemeral' - }); - - // Get the converted rule in Elasticsearch format using config from YAML - const config = { - backend: SIGMA_CLI_CONFIG.backend, - target: SIGMA_CLI_CONFIG.target, - format: SIGMA_CLI_CONFIG.format - }; - - logger.info(`${FILE_NAME}: Converting rule ${ruleId} for SIEM export`); - const conversionResult = await convertRuleToBackend(ruleId, config); - - if (!conversionResult.success) { - logger.error(`${FILE_NAME}: Rule conversion failed: ${conversionResult.message}`); - await respond({ - text: `Error: Failed to convert rule for SIEM: ${conversionResult.message}`, - replace_original: false, - response_type: 'ephemeral' - }); - return; - } - - // Parse the converted rule JSON - let rulePayload; - try { - rulePayload = JSON.parse(conversionResult.output); - - // Add required fields if not present - rulePayload.rule_id = rulePayload.rule_id || ruleId; - rulePayload.from = rulePayload.from || "now-360s"; - rulePayload.to = rulePayload.to || "now"; - rulePayload.interval = rulePayload.interval || "5m"; - - // Make sure required fields are present - if (!rulePayload.name) { - rulePayload.name = conversionResult.rule?.title || `Sigma Rule ${ruleId}`; - } - - if (!rulePayload.description) { - rulePayload.description = conversionResult.rule?.description || - `Converted from Sigma rule: ${ruleId}`; - } - - if (!rulePayload.risk_score) { - // Map Sigma level to risk score - const levelMap = { - 'critical': 90, - 'high': 73, - 'medium': 50, - 'low': 25, - 'informational': 10 - }; - - rulePayload.risk_score = levelMap[conversionResult.rule?.level] || 50; - } - - if (!rulePayload.severity) { - rulePayload.severity = conversionResult.rule?.level || 'medium'; - } - - if (!rulePayload.enabled) { - rulePayload.enabled = true; - } - - } catch (parseError) { - logger.error(`${FILE_NAME}: Failed to parse converted rule JSON: ${parseError.message}`); - await respond({ - text: `Error: The converted rule is not valid JSON: ${parseError.message}`, - replace_original: false, - response_type: 'ephemeral' - }); - return; - } - - // Send the rule to Elasticsearch using api service - try { - const result = await sendRuleToSiem(rulePayload); - - if (result.success) { - logger.info(`${FILE_NAME}: Successfully sent rule ${ruleId} to SIEM`); - await respond({ - text: `✅ Success! Rule "${rulePayload.name}" has been added to your Elasticsearch SIEM.`, - replace_original: false, - response_type: 'in_channel' - }); - } else { - logger.error(`${FILE_NAME}: Error sending rule to SIEM: ${result.message}`); - await respond({ - text: `Error: Failed to add rule to SIEM: ${result.message}`, - replace_original: false, - response_type: 'ephemeral' - }); - } - } catch (error) { - await handleError(error, `${FILE_NAME}: send_sigma_rule_to_siem action`, respond, { - replaceOriginal: false - }); - } - } catch (error) { - await handleError(error, `${FILE_NAME}: send_sigma_rule_to_siem action`, respond, { - replaceOriginal: false - }); - } - }); - - // Handle View YAML button clicks - app.action('view_yaml', async ({ body, ack, respond }) => { - logger.info(`${FILE_NAME}: VIEW_YAML ACTION TRIGGERED`); - try { - await ack(); - logger.debug(`${FILE_NAME}: View YAML action received: ${JSON.stringify(body.actions)}`); - - if (!body || !body.actions || !body.actions[0] || !body.actions[0].value) { - logger.error(`${FILE_NAME}: Invalid action payload: missing rule ID`); - await respond({ - text: 'Error: Could not determine which rule to get YAML for', - replace_original: false - }); - return; - } - - // Extract rule ID from button value - // Handle both formats: direct ID from search results or view_yaml_{ruleId} from details view - let ruleId = body.actions[0].value; - if (ruleId.startsWith('view_yaml_')) { - ruleId = ruleId.replace('view_yaml_', ''); - } - - logger.info(`${FILE_NAME}: View YAML button clicked for rule: ${ruleId}`); - - // Get Sigma rule YAML - const result = await getSigmaRuleYaml(ruleId); - logger.debug(`${FILE_NAME}: YAML retrieval result: ${JSON.stringify(result, null, 2)}`); - - if (!result.success) { - logger.error(`${FILE_NAME}: Rule YAML retrieval failed: ${result.message}`); - await respond({ - text: `Error: ${result.message}`, - replace_original: false - }); - return; - } - - logger.info(`${FILE_NAME}: Rule ${ruleId} YAML retrieved successfully via button click`); - - // Use the module to generate blocks - const blocks = getYamlViewBlocks(ruleId, result.yaml || ''); - - // Respond with the YAML content - await respond({ - blocks: blocks, - replace_original: false - }); - } catch (error) { - await handleError(error, `${FILE_NAME}: View YAML action`, respond, { - replaceOriginal: false - }); - } - }); - - // Handle convert_rule_to_siem button clicks - app.action('convert_rule_to_siem', async ({ body, ack, respond }) => { - try { - await ack(); - logger.debug(`${FILE_NAME}: convert_rule_to_siem action received: ${JSON.stringify(body.actions)}`); - - if (!body || !body.actions || !body.actions[0] || !body.actions[0].value) { - logger.error(`${FILE_NAME}: Invalid action payload: missing rule ID`); - await respond({ - text: 'Error: Could not determine which rule to convert', - replace_original: false - }); - return; - } - - // Extract rule ID from button value - const ruleId = body.actions[0].value.replace('convert_rule_to_siem_', ''); - logger.info(`${FILE_NAME}: convert_rule_to_siem button clicked for rule: ${ruleId}`); - - const config = { - backend: 'lucene', - target: 'ecs_windows', - format: 'siem_rule_ndjson' - }; - - await processRuleConversion(ruleId, config, respond, false, 'in_channel'); - } catch (error) { - await handleError(error, `${FILE_NAME}: convert_rule_to_siem action`, respond, { - replaceOriginal: false - }); - } - }); - - - // Handle "View Rule Details" button clicks from search results - app.action('view_rule_details', async ({ body, ack, respond }) => { - logger.info(`${FILE_NAME}: VIEW_RULE_DETAILS ACTION TRIGGERED`); - try { - await ack(); - logger.debug(`${FILE_NAME}: View Rule Details action received: ${JSON.stringify(body.actions)}`); - - if (!body || !body.actions || !body.actions[0] || !body.actions[0].value) { - logger.error(`${FILE_NAME}: Invalid action payload: missing rule ID`); - await respond({ - text: 'Error: Could not determine which rule to explain', - replace_original: false - }); - return; - } - - const ruleId = body.actions[0].value; - logger.info(`${FILE_NAME}: Rule details button clicked for rule ID: ${ruleId}`); - - // Inform user we're processing - await respond({ - text: `Processing details for rule ${ruleId}...`, - replace_original: false, - response_type: 'ephemeral' - }); - - await processRuleDetails(ruleId, respond, false, 'in_channel'); - } catch (error) { - await handleError(error, `${FILE_NAME}: View rule details action`, respond, { - replaceOriginal: false - }); - } - }); - - // Handle pagination button clicks - app.action('search_prev_page', async ({ body, ack, respond }) => { - await handlePaginationAction(body, ack, respond); - }); - - app.action('search_next_page', async ({ body, ack, respond }) => { - await handlePaginationAction(body, ack, respond); - }); - - logger.info(`${FILE_NAME}: All sigma action handlers registered successfully`); - - // Handle space selection button click - app.action('select_space_for_rule', async ({ body, ack, respond }) => { - try { - await ack(); - logger.debug(`${FILE_NAME}: select_space_for_rule action received: ${JSON.stringify(body.actions)}`); - - if (!body || !body.actions || !body.actions[0] || !body.actions[0].value) { - logger.error(`${FILE_NAME}: Invalid action payload: missing rule ID`); - await respond({ - text: 'Error: Could not determine which rule to select space for', - replace_original: false, - response_type: 'ephemeral' - }); - return; - } - - // Extract rule ID from value - const actionValue = body.actions[0].value; - const ruleId = actionValue.replace('select_space_for_rule_', ''); - - // Get rule information to display in the space selection message - const explainResult = await explainSigmaRule(ruleId); - const ruleInfo = explainResult.success ? explainResult.explanation : { title: ruleId }; - - // Generate blocks for space selection - const blocks = getSpaceSelectionBlocks(ruleId, ruleInfo); - - // Show space selection options - await respond({ - blocks: blocks, - replace_original: false, - response_type: 'ephemeral' - }); - - } catch (error) { - await handleError(error, `${FILE_NAME}: select_space_for_rule action`, respond, { - replaceOriginal: false - }); - } - }); - - // Handle space selection cancel button - app.action('cancel_space_selection', async ({ body, ack, respond }) => { - try { - await ack(); - await respond({ - text: 'Space selection cancelled.', - replace_original: false, - response_type: 'ephemeral' - }); - } catch (error) { - await handleError(error, `${FILE_NAME}: cancel_space_selection action`, respond, { - replaceOriginal: false - }); - } - }); - - // Dynamic handler for all space selection buttons - // This uses a pattern matcher to match any action ID that starts with "send_rule_to_space_" - app.action(/^send_rule_to_space_(.*)$/, async ({ body, action, ack, respond }) => { - try { - await ack(); - logger.debug(`${FILE_NAME}: Space selection action received: ${JSON.stringify(action)}`); - - // Extract rule ID and space ID from the action value - const actionValue = action.value; - const parts = actionValue.split('_'); - const spaceId = parts.pop(); // Last part is the space ID - const ruleId = actionValue.match(/send_rule_to_space_(.+)_/)[1]; // Extract full UUID - - logger.info(`${FILE_NAME}: Selected space ${spaceId} for rule ${ruleId}`); - - - // Get space info - const spaces = getAllSpaces(); - const selectedSpace = spaces.find(s => s.id === spaceId); - - if (!selectedSpace) { - logger.error(`${FILE_NAME}: Space not found: ${spaceId}`); - await respond({ - text: `Error: Space "${spaceId}" not found in configuration`, - replace_original: false, - response_type: 'ephemeral' - }); - return; - } - - // Inform user that processing is happening - await respond({ - text: `Sending rule ${ruleId} to ${selectedSpace.emoji || ''} ${selectedSpace.name} space...`, - replace_original: false, - response_type: 'ephemeral' - }); - - // Get the converted rule in Elasticsearch format - const config = { - backend: SIGMA_CLI_CONFIG.backend, - target: SIGMA_CLI_CONFIG.target, - format: SIGMA_CLI_CONFIG.format - }; - - logger.info(`${FILE_NAME}: Converting rule ${ruleId} for SIEM export to space ${spaceId}`); - const conversionResult = await convertRuleToBackend(ruleId, config); - - if (!conversionResult.success) { - logger.error(`${FILE_NAME}: Rule conversion failed: ${conversionResult.message}`); - await respond({ - text: `Error: Failed to convert rule for SIEM: ${conversionResult.message}`, - replace_original: false, - response_type: 'ephemeral' - }); - return; - } - - // Parse the converted rule JSON - let rulePayload; - try { - rulePayload = JSON.parse(conversionResult.output); - - // Add required fields if not present - rulePayload.rule_id = rulePayload.rule_id || ruleId; - rulePayload.from = rulePayload.from || "now-360s"; - rulePayload.to = rulePayload.to || "now"; - rulePayload.interval = rulePayload.interval || "5m"; - - // Set index pattern from space configuration if available - if (selectedSpace.indexPattern) { - rulePayload.index = Array.isArray(selectedSpace.indexPattern) - ? selectedSpace.indexPattern - : [selectedSpace.indexPattern]; - logger.debug(`${FILE_NAME}: Setting index pattern from space config: ${JSON.stringify(rulePayload.index)}`); - } - - // Make sure required fields are present - if (!rulePayload.name) { - rulePayload.name = conversionResult.rule?.title || `Sigma Rule ${ruleId}`; - } - - if (!rulePayload.description) { - rulePayload.description = conversionResult.rule?.description || - `Converted from Sigma rule: ${ruleId}`; - } - - if (!rulePayload.risk_score) { - // Map Sigma level to risk score - const levelMap = { - 'critical': 90, - 'high': 73, - 'medium': 50, - 'low': 25, - 'informational': 10 - }; - - rulePayload.risk_score = levelMap[conversionResult.rule?.level] || 50; - } - - if (!rulePayload.severity) { - rulePayload.severity = conversionResult.rule?.level || 'medium'; - } - - if (!rulePayload.enabled) { - rulePayload.enabled = true; - } - - } catch (parseError) { - logger.error(`${FILE_NAME}: Failed to parse converted rule JSON: ${parseError.message}`); - await respond({ - text: `Error: The converted rule is not valid JSON: ${parseError.message}`, - replace_original: false, - response_type: 'ephemeral' - }); - return; - } - - // Send the rule to the selected Elasticsearch space - try { - const result = await sendRuleToSiem(rulePayload, spaceId); - - if (result.success) { - logger.info(`${FILE_NAME}: Successfully sent rule ${ruleId} to space ${spaceId}`); - await respond({ - text: `✅ Success! Rule "${rulePayload.name}" has been added to the ${selectedSpace.emoji || ''} ${selectedSpace.name} space in Elasticsearch.`, - replace_original: false, - response_type: 'in_channel' - }); - } else { - logger.error(`${FILE_NAME}: Error sending rule to SIEM: ${result.message}`); - await respond({ - text: `Error: Failed to add rule to the ${selectedSpace.name} space: ${result.message}`, - replace_original: false, - response_type: 'ephemeral' - }); - } - } catch (error) { - await handleError(error, `${FILE_NAME}: send_rule_to_space action`, respond, { - replaceOriginal: false - }); - } - } catch (error) { - await handleError(error, `${FILE_NAME}: send_rule_to_space action`, respond, { - replaceOriginal: false - }); - } - }); -}; - - -module.exports = { - registerActionHandlers, - processRuleDetails, - processRuleConversion -}; \ No newline at end of file diff --git a/src/handlers/sigma/sigma_create_handler.js b/src/handlers/sigma/sigma_create_handler.js index 48384a1..1965f1c 100644 --- a/src/handlers/sigma/sigma_create_handler.js +++ b/src/handlers/sigma/sigma_create_handler.js @@ -2,11 +2,11 @@ * sigma_create_handler.js * * Handles Sigma rule conversion requests from Slack commands - * Action handlers moved to sigma_action_handlers.js + * Action handlers moved to sigma_action_core.js */ const logger = require('../../utils/logger'); const { handleError } = require('../../utils/error_handler'); -const { processRuleConversion } = require('./sigma_action_handlers'); +const { processRuleConversion } = require('./actions/sigma_action_core'); const { SIGMA_CLI_CONFIG } = require('../../config/appConfig'); const FILE_NAME = 'sigma_create_handler.js'; diff --git a/src/handlers/sigma/sigma_details_handler.js b/src/handlers/sigma/sigma_details_handler.js index ce5bbb7..55844b3 100644 --- a/src/handlers/sigma/sigma_details_handler.js +++ b/src/handlers/sigma/sigma_details_handler.js @@ -7,10 +7,8 @@ const logger = require('../../utils/logger'); const { handleError } = require('../../utils/error_handler'); const { explainSigmaRule } = require('../../services/sigma/sigma_details_service'); -const { processRuleDetails } = require('./sigma_action_handlers'); - +const { processRuleDetails } = require('./actions/sigma_action_core'); const FILE_NAME = 'sigma_details_handler.js'; - /** * Handle the sigma-details command for Sigma rules * @@ -50,7 +48,6 @@ const handleCommand = async (command, respond) => { }); } }; - module.exports = { handleCommand }; \ No newline at end of file -- 2.39.5 From 181eade8c43f10c351b23499d2c1d513e44f04f3 Mon Sep 17 00:00:00 2001 From: Charlotte Croce Date: Fri, 18 Apr 2025 13:49:10 -0400 Subject: [PATCH 04/15] create NLP command for details --- src/app.js | 29 ++-- src/handlers/fylgja_command_handler.js | 142 ++++++++++++++++++++ src/handlers/sigma/sigma_stats_handler.js | 2 +- src/lang/command_parser.js | 105 +++++++++++++++ src/lang/command_patterns.js | 78 +++++++++++ src/services/sigma/sigma_details_service.js | 2 +- 6 files changed, 348 insertions(+), 10 deletions(-) create mode 100644 src/handlers/fylgja_command_handler.js create mode 100644 src/lang/command_parser.js create mode 100644 src/lang/command_patterns.js diff --git a/src/app.js b/src/app.js index f2cc711..1a17bb3 100644 --- a/src/app.js +++ b/src/app.js @@ -3,7 +3,7 @@ * * Main application file for Fylgja Slack bot * Initializes the Slack Bolt app with custom ExpressReceiver Registers command handlers - * + * Now supports the universal /fylgja command */ const { App, ExpressReceiver } = require('@slack/bolt'); const fs = require('fs'); @@ -13,16 +13,14 @@ const { SIGMA_CLI_PATH, SIGMA_CLI_CONFIG, SLACK_CONFIG } = require('./config/app const { getFileName } = require('./utils/file_utils'); const FILE_NAME = getFileName(__filename); -// Import individual command handlers +// Import the unified fylgja command handler +const fylgjaCommandHandler = require('./handlers/fylgja_command_handler'); + const sigmaDetailsHandler = require('./handlers/sigma/sigma_details_handler'); const sigmaSearchHandler = require('./handlers/sigma/sigma_search_handler'); const sigmaCreateHandler = require('./handlers/sigma/sigma_create_handler'); // Import the action registry const sigmaActionRegistry = require('./handlers/sigma/actions/sigma_action_registry'); -//const configCommand = require('./commands/config/index.js'); -//const alertsCommand = require('./commands/alerts/index.js'); -//const caseCommand = require('./commands/case/index.js'); -//const statsCommand = require('./commands/stats/index.js'); // Verify sigma-cli is installed if (!fs.existsSync(SIGMA_CLI_PATH)) { @@ -49,8 +47,23 @@ const app = new App({ receiver: expressReceiver }); -// Register individual command handlers for all sigma commands -logger.info('Registering command handlers'); +// Register the unified fylgja command handler +logger.info('Registering unified fylgja command handler'); +app.command('/fylgja', async ({ command, ack, respond }) => { + try { + await ack(); + logger.info(`Received fylgja command: ${command.text}`); + await fylgjaCommandHandler.handleCommand(command, respond); + } catch (error) { + logger.error(`Error handling fylgja command: ${error.message}`); + logger.debug(`Error stack: ${error.stack}`); + await respond({ + text: `An error occurred: ${error.message}`, + response_type: 'ephemeral' + }); + } +}); + // Register sigma command handlers directly app.command('/sigma-create', async ({ command, ack, respond }) => { diff --git a/src/handlers/fylgja_command_handler.js b/src/handlers/fylgja_command_handler.js new file mode 100644 index 0000000..191e0b2 --- /dev/null +++ b/src/handlers/fylgja_command_handler.js @@ -0,0 +1,142 @@ +/** + * fylgja_command_handler.js + * + * Unified command handler for the Fylgja Slack bot. + * Processes natural language commands and routes to appropriate handlers. + */ +const logger = require('../utils/logger'); +const { handleError } = require('../utils/error_handler'); +const FILE_NAME = 'fylgja_command_handler.js'; + +// Import command handlers +const sigmaDetailsHandler = require('./sigma/sigma_details_handler'); +const sigmaSearchHandler = require('./sigma/sigma_search_handler'); +const sigmaCreateHandler = require('./sigma/sigma_create_handler'); +const sigmaStatsHandler = require('./sigma/sigma_stats_handler'); + +// Import language processing utilities +const commandParser = require('../lang/command_parser'); + +/** + * Handle the universal fylgja command + * + * @param {Object} command - The Slack command object + * @param {Function} respond - Function to send response back to Slack + */ +const handleCommand = async (command, respond) => { + try { + if (!command || !command.text) { + logger.warn(`${FILE_NAME}: Empty command received for fylgja`); + await respond({ + text: 'Please provide a command. Try `/fylgja help` for available commands.', + response_type: 'ephemeral' + }); + return; + } + + logger.info(`${FILE_NAME}: Processing fylgja command: ${command.text}`); + + // Parse the natural language command + const parsedCommand = await commandParser.parseCommand(command.text); + + if (!parsedCommand.success) { + logger.warn(`${FILE_NAME}: Failed to parse command: ${command.text}`); + await respond({ + text: `I couldn't understand that command. ${parsedCommand.message || ''}`, + response_type: 'ephemeral' + }); + return; + } + + // Route to the appropriate handler based on the parsed command + await routeCommand(parsedCommand.command, command, respond); + + } catch (error) { + await handleError(error, `${FILE_NAME}: Command handler`, respond, { + responseType: 'ephemeral' + }); + } +}; + +/** + * Route the command to the appropriate handler + * + * @param {Object} parsedCommand - The parsed command object + * @param {Object} originalCommand - The original Slack command object + * @param {Function} respond - Function to send response back to Slack + */ +const routeCommand = async (parsedCommand, originalCommand, respond) => { + const { action, module, params } = parsedCommand; + + // Create a modified command object with the extracted parameters + const modifiedCommand = { + ...originalCommand, + text: params.join(' ') + }; + + // Log the routing decision + logger.debug(`${FILE_NAME}: Routing command - Action: ${action}, Module: ${module}, Params: ${JSON.stringify(params)}`); + + // Route to the appropriate handler + switch (`${module}:${action}`) { + case 'sigma:details': + case 'sigma:explain': + await sigmaDetailsHandler.handleCommand(modifiedCommand, respond); + break; + + case 'sigma:search': + await sigmaSearchHandler.handleCommand(modifiedCommand, respond); + break; + + case 'sigma:create': + await sigmaCreateHandler.handleCommand(modifiedCommand, respond); + break; + + case 'sigma:stats': + await sigmaStatsHandler.handleCommand(modifiedCommand, respond); + break; + + case 'help:general': + await showHelp(respond); + break; + + default: + logger.warn(`${FILE_NAME}: Unknown command combination: ${module}:${action}`); + await respond({ + text: `I don't know how to ${action} in ${module}. Try \`/fylgja help\` for available commands.`, + response_type: 'ephemeral' + }); + } +}; + +/** + * Show help information + * + * @param {Function} respond - Function to send response back to Slack + */ +const showHelp = async (respond) => { + await respond({ + text: "Here are some example commands you can use with Fylgja:", + blocks: [ + { + type: "section", + text: { + type: "mrkdwn", + text: "*Fylgja Commands*\nHere are some example commands you can use:" + } + }, + { + type: "section", + text: { + type: "mrkdwn", + text: "• `/fylgja explain rule from sigma where id=`\n• `/fylgja search sigma for `\n• `/fylgja create rule in sigma with `\n• `/fylgja show stats for sigma`" + } + } + ], + response_type: 'ephemeral' + }); +}; + +module.exports = { + handleCommand +}; \ No newline at end of file diff --git a/src/handlers/sigma/sigma_stats_handler.js b/src/handlers/sigma/sigma_stats_handler.js index a9c571c..5b49a63 100644 --- a/src/handlers/sigma/sigma_stats_handler.js +++ b/src/handlers/sigma/sigma_stats_handler.js @@ -7,7 +7,7 @@ const logger = require('../../utils/logger'); const { handleError } = require('../../utils/error_handler'); const { getSigmaStats } = require('../../services/sigma/sigma_stats_service'); -const { getStatsBlocks } = require('../../blocks/sigma_stats_block'); +const { getStatsBlocks } = require('../../blocks/sigma/sigma_stats_block'); const { getFileName } = require('../../utils/file_utils'); const FILE_NAME = getFileName(__filename); diff --git a/src/lang/command_parser.js b/src/lang/command_parser.js new file mode 100644 index 0000000..c24c6a6 --- /dev/null +++ b/src/lang/command_parser.js @@ -0,0 +1,105 @@ +/** + * command_parser.js + * + * Provides functionality for parsing commands for the Fylgja bot + */ +const logger = require('../utils/logger'); +const FILE_NAME = 'command_parser.js'; + +// Import language patterns and synonyms +const commandPatterns = require('./command_patterns'); + +/** + * Parse a natural language command into a structured command object + * + * @param {string} commandText - The natural language command text + * @returns {Promise} Result object with success flag and parsed command or error message + */ +const parseCommand = async (commandText) => { + try { + logger.debug(`${FILE_NAME}: Parsing command: ${commandText}`); + + if (!commandText || typeof commandText !== 'string') { + return { + success: false, + message: 'Empty or invalid command.' + }; + } + + // Convert to lowercase for case-insensitive matching + const normalizedCommand = commandText.toLowerCase().trim(); + + // TODO + // Handle help command separately + if (normalizedCommand === 'help') { + return { + success: true, + command: { + action: 'general', + module: 'help', + params: [] + } + }; + } + + // Try to match command against known patterns + for (const pattern of commandPatterns) { + const match = matchPattern(normalizedCommand, pattern); + if (match) { + logger.debug(`${FILE_NAME}: Command matched pattern: ${pattern.name}`); + return { + success: true, + command: match + }; + } + } + + // If we reach here, no pattern matched + logger.warn(`${FILE_NAME}: No pattern matched for command: ${commandText}`); + return { + success: false, + message: "I couldn't understand that command. Try `/fylgja help` for examples." + }; + } catch (error) { + logger.error(`${FILE_NAME}: Error parsing command: ${error.message}`); + logger.debug(`${FILE_NAME}: Error stack: ${error.stack}`); + return { + success: false, + message: `Error parsing command: ${error.message}` + }; + } +}; + +/** + * Match a command against a pattern + * + * @param {string} command - The normalized command text + * @param {Object} pattern - The pattern object to match against + * @returns {Object|null} Parsed command object or null if no match + */ +const matchPattern = (command, pattern) => { + // Check if the command matches the regex pattern + const match = pattern.regex.exec(command); + if (!match) { + return null; + } + + // Extract parameters based on the pattern's parameter mapping + const params = []; + for (const paramIndex of pattern.params) { + if (match[paramIndex]) { + params.push(match[paramIndex].trim()); + } + } + + // Return the structured command + return { + action: pattern.action, + module: pattern.module, + params + }; +}; + +module.exports = { + parseCommand +}; \ No newline at end of file diff --git a/src/lang/command_patterns.js b/src/lang/command_patterns.js new file mode 100644 index 0000000..599f272 --- /dev/null +++ b/src/lang/command_patterns.js @@ -0,0 +1,78 @@ +/** + * command_patterns.js + * + * Defines pattern matching rules for natural language commands + * Each pattern includes a regex and mapping for parameter extraction + */ + +/** + * Command patterns array + * Each pattern object contains: + * - name: A descriptive name for the pattern + * - regex: A regular expression to match the command + * - action: The action to perform (e.g., details, search) + * - module: The module to use (e.g., sigma, alerts) + * - params: Array of capturing group indices to extract parameters + */ +const commandPatterns = [ + // Sigma details patterns + { + name: 'sigma-details-direct', + regex: /^(explain|get|show|display|details|info|about)\s+(rule|detection)\s+(from\s+)?sigma\s+(where\s+)?(id=|id\s+is\s+|with\s+id\s+)(.+)$/i, + action: 'details', + module: 'sigma', + params: [6] // rule ID is in capturing group 6 + }, + { + name: 'sigma-details-simple', + regex: /^(details|explain)\s+(.+)$/i, + action: 'details', + module: 'sigma', + params: [2] // rule ID is in capturing group 2 + }, + + // Sigma search patterns + { + name: 'sigma-search', + regex: /^(search|find|look\s+for)\s+(rules|detections)?\s*(in|from)?\s*sigma\s+(for|where|with)?\s+(.+)$/i, + action: 'search', + module: 'sigma', + params: [5] // search query is in capturing group 5 + }, + { + name: 'sigma-search-simple', + regex: /^(search|find)\s+(.+)$/i, + action: 'search', + module: 'sigma', + params: [2] // search query is in capturing group 2 + }, + + // Sigma create patterns + { + name: 'sigma-create', + regex: /^(create|new|add)\s+(rule|detection)\s+(in|to|for)?\s*sigma\s+(with|using)?\s+(.+)$/i, + action: 'create', + module: 'sigma', + params: [5] // creation parameters in capturing group 5 + }, + + // Sigma stats patterns + { + name: 'sigma-stats', + regex: /^(stats|statistics|metrics|counts)\s+(for|about|on|of)?\s*sigma$/i, + action: 'stats', + module: 'sigma', + params: [] + }, + { + name: 'sigma-stats-show', + regex: /^(show|get|display)\s+(stats|statistics|metrics|counts)\s+(for|about|on|of)?\s*sigma$/i, + action: 'stats', + module: 'sigma', + params: [] + } + + // Additional command patterns for other modules can be added here + ]; + + module.exports = commandPatterns; \ No newline at end of file diff --git a/src/services/sigma/sigma_details_service.js b/src/services/sigma/sigma_details_service.js index dad7db1..2f43019 100644 --- a/src/services/sigma/sigma_details_service.js +++ b/src/services/sigma/sigma_details_service.js @@ -147,4 +147,4 @@ async function getSigmaRuleYaml(ruleId) { module.exports = { explainSigmaRule, getSigmaRuleYaml -}; \ No newline at end of file +}; -- 2.39.5 From d839089153f31f9f453d6aecfb0f56e6ffc33669 Mon Sep 17 00:00:00 2001 From: Charlotte Croce Date: Fri, 18 Apr 2025 14:03:22 -0400 Subject: [PATCH 05/15] add command patterns for convert rule and sigma-stats --- src/lang/command_patterns.js | 84 ++++++++++++++++++------------------ 1 file changed, 41 insertions(+), 43 deletions(-) diff --git a/src/lang/command_patterns.js b/src/lang/command_patterns.js index 599f272..22f3e98 100644 --- a/src/lang/command_patterns.js +++ b/src/lang/command_patterns.js @@ -17,62 +17,60 @@ const commandPatterns = [ // Sigma details patterns { - name: 'sigma-details-direct', - regex: /^(explain|get|show|display|details|info|about)\s+(rule|detection)\s+(from\s+)?sigma\s+(where\s+)?(id=|id\s+is\s+|with\s+id\s+)(.+)$/i, - action: 'details', - module: 'sigma', - params: [6] // rule ID is in capturing group 6 + name: 'sigma-details-direct', + regex: /^(explain|get|show|display|details|info|about)\s+(rule|detection)\s+(from\s+)?sigma\s+(where\s+)?(id=|id\s+is\s+|with\s+id\s+)(.+)$/i, + action: 'details', + module: 'sigma', + params: [6] // rule ID is in capturing group 6 }, { - name: 'sigma-details-simple', - regex: /^(details|explain)\s+(.+)$/i, - action: 'details', - module: 'sigma', - params: [2] // rule ID is in capturing group 2 + name: 'sigma-details-simple', + regex: /^(details|explain)\s+(.+)$/i, + action: 'details', + module: 'sigma', + params: [2] // rule ID is in capturing group 2 }, - + // Sigma search patterns { - name: 'sigma-search', - regex: /^(search|find|look\s+for)\s+(rules|detections)?\s*(in|from)?\s*sigma\s+(for|where|with)?\s+(.+)$/i, - action: 'search', - module: 'sigma', - params: [5] // search query is in capturing group 5 + name: 'sigma-search', + regex: /^(search|find|look\s+for)\s+(rules|detections)?\s*(in|from)?\s*sigma\s+(for|where|with)?\s+(.+)$/i, + action: 'search', + module: 'sigma', + params: [5] // search query is in capturing group 5 }, { - name: 'sigma-search-simple', - regex: /^(search|find)\s+(.+)$/i, - action: 'search', - module: 'sigma', - params: [2] // search query is in capturing group 2 + name: 'sigma-search-simple', + regex: /^(search|find)\s+(.+)$/i, + action: 'search', + module: 'sigma', + params: [2] // search query is in capturing group 2 }, - + // Sigma create patterns { - name: 'sigma-create', - regex: /^(create|new|add)\s+(rule|detection)\s+(in|to|for)?\s*sigma\s+(with|using)?\s+(.+)$/i, - action: 'create', - module: 'sigma', - params: [5] // creation parameters in capturing group 5 + name: 'sigma-create', + regex: /^(create|convert)\s+sigma\s+rule\s+where\s+id=(.+)$/i, + action: 'create', + module: 'sigma', + params: [2] // rule ID is in capturing group 2 }, - + // Sigma stats patterns { - name: 'sigma-stats', - regex: /^(stats|statistics|metrics|counts)\s+(for|about|on|of)?\s*sigma$/i, - action: 'stats', - module: 'sigma', - params: [] + name: 'sigma-stats-first', + regex: /^sigma\s+stats$/i, + action: 'stats', + module: 'sigma', + params: [] }, { - name: 'sigma-stats-show', - regex: /^(show|get|display)\s+(stats|statistics|metrics|counts)\s+(for|about|on|of)?\s*sigma$/i, - action: 'stats', - module: 'sigma', - params: [] + name: 'sigma-stats-second', + regex: /^stats\s+sigma$/i, + action: 'stats', + module: 'sigma', + params: [] } - - // Additional command patterns for other modules can be added here - ]; - - module.exports = commandPatterns; \ No newline at end of file +]; + +module.exports = commandPatterns; \ No newline at end of file -- 2.39.5 From 167829704adc3eec58ce6ab3084910783cb7442b Mon Sep 17 00:00:00 2001 From: Charlotte Croce Date: Fri, 18 Apr 2025 14:51:43 -0400 Subject: [PATCH 06/15] update search command to use FTS5 SQLite table for complex searches --- src/handlers/config/config_handler.js | 88 --- src/handlers/fylgja_command_handler.js | 291 ++++++---- src/handlers/sigma/sigma_search_handler.js | 168 +++++- src/lang/command_patterns.js | 17 +- src/lang/query_parser.js | 197 +++++++ src/services/sigma/sigma_search_service.js | 111 +++- src/sigma_db/sigma_db_initialize.js | 130 ++++- src/sigma_db/sigma_db_queries.js | 624 ++++++++++++++++++++- 8 files changed, 1359 insertions(+), 267 deletions(-) create mode 100644 src/lang/query_parser.js diff --git a/src/handlers/config/config_handler.js b/src/handlers/config/config_handler.js index ff58672..e69de29 100644 --- a/src/handlers/config/config_handler.js +++ b/src/handlers/config/config_handler.js @@ -1,88 +0,0 @@ -// -// config_handler.js -// handle the /sigma-config command -// -const util = require('util'); -const { exec } = require('child_process'); -const { SIGMA_CLI_PATH } = require('../../config/constants'); -const { loadConfig, updateConfig } = require('../../config/config-manager'); -const { updateSigmaDatabase } = require('../../services/sigma/sigma_repository_service'); -const logger = require('../../utils/logger'); - -// Promisify exec for async/await usage -const execPromise = util.promisify(exec); - -module.exports = (app) => { - app.command('/sigma-config', async ({ command, ack, respond }) => { - await ack(); - logger.info(`Sigma config command received: ${command.text}`); - - const args = command.text.split(' '); - - if (args.length === 0 || args[0] === '') { - // Display current configuration - const config = loadConfig(); - logger.info('Displaying current configuration'); - await respond(`Current configuration:\nSIEM: ${config.siem}\nLanguage: ${config.lang}\nOutput: ${config.output}`); - return; - } - - const configType = args[0]; - - if (configType === 'update') { - logger.info('Starting database update from command'); - try { - await respond('Updating Sigma database... This may take a moment.'); - await updateSigmaDatabase(); - logger.info('Database update completed from command'); - await respond('Sigma database updated successfully'); - } catch (error) { - logger.error(`Database update failed: ${error.message}`); - await respond(`Error updating Sigma database: ${error.message}`); - } - return; - } - - if (args.length < 2) { - logger.warn(`Invalid config command format: ${command.text}`); - await respond(`Invalid command format. Usage: /sigma-config ${configType} [value]`); - return; - } - - const configValue = args[1]; - const config = loadConfig(); - - if (configType === 'siem') { - // Verify the SIEM backend is installed - logger.info(`Attempting to change SIEM to: ${configValue}`); - try { - await execPromise(`${SIGMA_CLI_PATH} list targets | grep ${configValue}`); - updateConfig('siem', configValue); - logger.info(`SIEM configuration updated to: ${configValue}`); - await respond(`SIEM configuration updated to: ${configValue}`); - } catch (error) { - logger.error(`SIEM backend '${configValue}' not found or not installed`); - await respond(`Error: SIEM backend '${configValue}' not found or not installed. Please install it with: sigma plugin install ${configValue}`); - } - } else if (configType === 'lang') { - logger.info(`Changing language to: ${configValue}`); - updateConfig('lang', configValue); - await respond(`Language configuration updated to: ${configValue}`); - } else if (configType === 'output') { - // Check if output format is supported by the current backend - logger.info(`Attempting to change output format to: ${configValue}`); - try { - await execPromise(`${SIGMA_CLI_PATH} list formats ${config.siem} | grep ${configValue}`); - updateConfig('output', configValue); - logger.info(`Output configuration updated to: ${configValue}`); - await respond(`Output configuration updated to: ${configValue}`); - } catch (error) { - logger.error(`Output format '${configValue}' not supported by SIEM backend '${config.siem}'`); - await respond(`Error: Output format '${configValue}' not supported by SIEM backend '${config.siem}'. Run 'sigma list formats ${config.siem}' to see available formats.`); - } - } else { - logger.warn(`Unknown configuration type: ${configType}`); - await respond(`Unknown configuration type: ${configType}. Available types: siem, lang, output, update`); - } - }); -}; \ No newline at end of file diff --git a/src/handlers/fylgja_command_handler.js b/src/handlers/fylgja_command_handler.js index 191e0b2..f9b2b37 100644 --- a/src/handlers/fylgja_command_handler.js +++ b/src/handlers/fylgja_command_handler.js @@ -1,142 +1,209 @@ /** * fylgja_command_handler.js * - * Unified command handler for the Fylgja Slack bot. - * Processes natural language commands and routes to appropriate handlers. + * Main handler for the /fylgja slash command + * Parses natural language commands and routes to appropriate handlers */ + const logger = require('../utils/logger'); +const { parseCommand } = require('../lang/command_parser'); const { handleError } = require('../utils/error_handler'); +const { handleCommand: handleSigmaSearch, handleComplexSearch } = require('./sigma/sigma_search_handler'); +const { handleCommand: handleSigmaDetails } = require('./sigma/sigma_details_handler'); +const { handleCommand: handleSigmaStats } = require('./sigma/sigma_stats_handler'); +const { handleCommand: handleSigmaCreate } = require('./sigma/sigma_create_handler'); +const { handleCommand: handleAlerts } = require('./alerts/alerts_handler'); +const { handleCommand: handleCase } = require('./case/case_handler'); +const { handleCommand: handleConfig } = require('./config/config_handler'); +const { handleCommand: handleStats } = require('./stats/stats_handler'); + const FILE_NAME = 'fylgja_command_handler.js'; -// Import command handlers -const sigmaDetailsHandler = require('./sigma/sigma_details_handler'); -const sigmaSearchHandler = require('./sigma/sigma_search_handler'); -const sigmaCreateHandler = require('./sigma/sigma_create_handler'); -const sigmaStatsHandler = require('./sigma/sigma_stats_handler'); - -// Import language processing utilities -const commandParser = require('../lang/command_parser'); - /** - * Handle the universal fylgja command + * Main handler for the /fylgja command + * Parses natural language input and routes to appropriate module handlers * * @param {Object} command - The Slack command object * @param {Function} respond - Function to send response back to Slack */ const handleCommand = async (command, respond) => { - try { - if (!command || !command.text) { - logger.warn(`${FILE_NAME}: Empty command received for fylgja`); - await respond({ - text: 'Please provide a command. Try `/fylgja help` for available commands.', - response_type: 'ephemeral' - }); - return; - } + try { + logger.info(`${FILE_NAME}: Received command: ${command.text}`); - logger.info(`${FILE_NAME}: Processing fylgja command: ${command.text}`); - - // Parse the natural language command - const parsedCommand = await commandParser.parseCommand(command.text); - - if (!parsedCommand.success) { - logger.warn(`${FILE_NAME}: Failed to parse command: ${command.text}`); - await respond({ - text: `I couldn't understand that command. ${parsedCommand.message || ''}`, - response_type: 'ephemeral' - }); - return; + if (!command.text.trim()) { + logger.warn(`${FILE_NAME}: Empty command received`); + await respond({ + text: "Please provide a command. Try `/fylgja help` for usage examples.", + response_type: 'ephemeral' + }); + return; + } + + // Parse the natural language command + const parsedCommand = await parseCommand(command.text); + logger.debug(`${FILE_NAME}: Parsed command result: ${JSON.stringify(parsedCommand)}`); + + if (!parsedCommand.success) { + logger.warn(`${FILE_NAME}: Command parsing failed: ${parsedCommand.message}`); + await respond({ + text: parsedCommand.message || "I couldn't understand that command. Try `/fylgja help` for examples.", + response_type: 'ephemeral' + }); + return; + } + + // Extract the structured command + const { action, module, params } = parsedCommand.command; + logger.info(`${FILE_NAME}: Routing command - Module: ${module}, Action: ${action}`); + + // Route to the appropriate handler based on module and action + switch (module) { + case 'sigma': + await handleSigmaCommand(action, params, command, respond); + break; + + case 'alerts': + await handleAlerts(command, respond); + break; + + case 'case': + await handleCase(command, respond); + break; + + case 'config': + await handleConfig(command, respond); + break; + + case 'stats': + await handleStats(command, respond); + break; + + case 'help': + await handleHelpCommand(respond); + break; + + default: + logger.warn(`${FILE_NAME}: Unknown module: ${module}`); + await respond({ + text: `Unknown command module: ${module}. Try \`/fylgja help\` for usage examples.`, + response_type: 'ephemeral' + }); + } + } catch (error) { + await handleError(error, `${FILE_NAME}: Command handler`, respond, { + responseType: 'ephemeral' + }); } - - // Route to the appropriate handler based on the parsed command - await routeCommand(parsedCommand.command, command, respond); - - } catch (error) { - await handleError(error, `${FILE_NAME}: Command handler`, respond, { - responseType: 'ephemeral' - }); - } }; /** - * Route the command to the appropriate handler + * Handle Sigma-related commands * - * @param {Object} parsedCommand - The parsed command object - * @param {Object} originalCommand - The original Slack command object - * @param {Function} respond - Function to send response back to Slack + * @param {string} action - The action to perform + * @param {Array} params - Command parameters + * @param {Object} command - The original Slack command + * @param {Function} respond - Function to send response */ -const routeCommand = async (parsedCommand, originalCommand, respond) => { - const { action, module, params } = parsedCommand; - - // Create a modified command object with the extracted parameters - const modifiedCommand = { - ...originalCommand, - text: params.join(' ') - }; - - // Log the routing decision - logger.debug(`${FILE_NAME}: Routing command - Action: ${action}, Module: ${module}, Params: ${JSON.stringify(params)}`); - - // Route to the appropriate handler - switch (`${module}:${action}`) { - case 'sigma:details': - case 'sigma:explain': - await sigmaDetailsHandler.handleCommand(modifiedCommand, respond); - break; - - case 'sigma:search': - await sigmaSearchHandler.handleCommand(modifiedCommand, respond); - break; - - case 'sigma:create': - await sigmaCreateHandler.handleCommand(modifiedCommand, respond); - break; - - case 'sigma:stats': - await sigmaStatsHandler.handleCommand(modifiedCommand, respond); - break; - - case 'help:general': - await showHelp(respond); - break; - - default: - logger.warn(`${FILE_NAME}: Unknown command combination: ${module}:${action}`); - await respond({ - text: `I don't know how to ${action} in ${module}. Try \`/fylgja help\` for available commands.`, - response_type: 'ephemeral' - }); - } +const handleSigmaCommand = async (action, params, command, respond) => { + logger.debug(`${FILE_NAME}: Handling Sigma command - Action: ${action}, Params: ${JSON.stringify(params)}`); + + try { + switch (action) { + case 'search': + // Update the command object with the keyword parameter + command.text = params[0] || ''; + await handleSigmaSearch(command, respond); + break; + + case 'complexSearch': + // Update the command object with the complex query + command.text = params[0] || ''; + await handleComplexSearch(command, respond); + break; + + case 'details': + // Update the command object with the rule ID parameter + command.text = params[0] || ''; + await handleSigmaDetails(command, respond); + break; + + case 'stats': + await handleSigmaStats(command, respond); + break; + + case 'create': + // Update the command object with the rule ID parameter + command.text = params[0] || ''; + await handleSigmaCreate(command, respond); + break; + + default: + logger.warn(`${FILE_NAME}: Unknown Sigma action: ${action}`); + await respond({ + text: `Unknown Sigma action: ${action}. Try \`/fylgja help\` for usage examples.`, + response_type: 'ephemeral' + }); + } + } catch (error) { + await handleError(error, `${FILE_NAME}: Sigma command handler`, respond, { + responseType: 'ephemeral' + }); + } }; /** - * Show help information + * Handle help command * - * @param {Function} respond - Function to send response back to Slack + * @param {Function} respond - Function to send response */ -const showHelp = async (respond) => { - await respond({ - text: "Here are some example commands you can use with Fylgja:", - blocks: [ - { - type: "section", - text: { - type: "mrkdwn", - text: "*Fylgja Commands*\nHere are some example commands you can use:" - } - }, - { - type: "section", - text: { - type: "mrkdwn", - text: "• `/fylgja explain rule from sigma where id=`\n• `/fylgja search sigma for `\n• `/fylgja create rule in sigma with `\n• `/fylgja show stats for sigma`" - } - } - ], - response_type: 'ephemeral' - }); +const handleHelpCommand = async (respond) => { + try { + const helpText = ` +*Fylgja Command Help* + +*Basic Commands:* +• \`/fylgja search \` - Search for Sigma rules by keyword +• \`/fylgja details \` - Get details about a specific Sigma rule +• \`/fylgja stats\` - Get statistics about Sigma rules database + +*Advanced Search Commands:* +• \`/fylgja search sigma rules where title contains "ransomware"\` - Search by title +• \`/fylgja find rules where tags include privilege_escalation\` - Search by tags +• \`/fylgja search rules where logsource.category == "process_creation"\` - Search by log source +• \`/fylgja find rules where modified after 2024-01-01\` - Search by modification date +• \`/fylgja search where level is "high" and tags include "attack.t1055"\` - Combined search + +*Supported Conditions:* +• Title: \`title contains "text"\` +• Description: \`description contains "text"\` +• Log Source: \`logsource.category == "value"\`, \`logsource.product == "value"\` +• Tags: \`tags include "value"\` +• Dates: \`modified after YYYY-MM-DD\`, \`modified before YYYY-MM-DD\` +• Author: \`author is "name"\` +• Level: \`level is "high"\` + +*Logical Operators:* +• AND: \`condition1 AND condition2\` +• OR: \`condition1 OR condition2\` + +*Pagination:* +• Add \`page=N\` to see page N of results +• Add \`limit=N\` to change number of results per page + +For more information, visit the Fylgja documentation. + `; + + await respond({ + text: helpText, + response_type: 'ephemeral' + }); + } catch (error) { + await handleError(error, `${FILE_NAME}: Help command handler`, respond, { + responseType: 'ephemeral' + }); + } }; module.exports = { - handleCommand + handleCommand }; \ No newline at end of file diff --git a/src/handlers/sigma/sigma_search_handler.js b/src/handlers/sigma/sigma_search_handler.js index e161b7d..f6d9e17 100644 --- a/src/handlers/sigma/sigma_search_handler.js +++ b/src/handlers/sigma/sigma_search_handler.js @@ -3,11 +3,11 @@ * * Handles Sigma rule search requests from Slack commands */ -const { searchSigmaRules } = require('../../services/sigma/sigma_search_service'); + +const { searchSigmaRules, searchSigmaRulesComplex } = require('../../services/sigma/sigma_search_service'); const logger = require('../../utils/logger'); const { handleError } = require('../../utils/error_handler'); const { getSearchResultBlocks } = require('../../blocks/sigma/sigma_search_results_block'); - const { getFileName } = require('../../utils/file_utils'); const FILE_NAME = getFileName(__filename); @@ -24,18 +24,18 @@ const MAX_RESULTS_THRESHOLD = 99; const handleCommand = async (command, respond) => { try { logger.debug(`${FILE_NAME}: Processing sigma-search command: ${JSON.stringify(command.text)}`); - + if (!command || !command.text) { logger.warn(`${FILE_NAME}: Empty command received for sigma-search`); await respond('Invalid command. Usage: /sigma-search [keyword]'); return; } - + // Extract search keyword and check for pagination parameters let keyword = command.text.trim(); let page = 1; let pageSize = MAX_RESULTS_PER_PAGE; - + // Check for pagination format: keyword page=X const pagingMatch = keyword.match(/(.+)\s+page=(\d+)$/i); if (pagingMatch) { @@ -43,7 +43,7 @@ const handleCommand = async (command, respond) => { page = parseInt(pagingMatch[2], 10) || 1; logger.debug(`${FILE_NAME}: Detected pagination request: "${keyword}" page ${page}`); } - + // Check for page size format: keyword limit=X const limitMatch = keyword.match(/(.+)\s+limit=(\d+)$/i); if (limitMatch) { @@ -53,29 +53,27 @@ const handleCommand = async (command, respond) => { pageSize = Math.min(Math.max(pageSize, 1), 100); logger.debug(`${FILE_NAME}: Detected page size request: "${keyword}" limit ${pageSize}`); } - + if (!keyword) { logger.warn(`${FILE_NAME}: Missing keyword in sigma-search command`); await respond('Invalid command: missing keyword. Usage: /sigma-search [keyword]'); return; } - + logger.info(`${FILE_NAME}: Searching for rules with keyword: ${keyword} (page ${page}, size ${pageSize})`); logger.debug(`${FILE_NAME}: Search keyword length: ${keyword.length}`); - + await respond({ text: 'Searching for rules... This may take a moment.', response_type: 'ephemeral' }); - + // Search for rules using the service function with pagination const searchResult = await searchSigmaRules(keyword, page, pageSize); - logger.debug(`${FILE_NAME}: Search result status: ${searchResult.success}`); logger.debug(`${FILE_NAME}: Found ${searchResult.results?.length || 0} results out of ${searchResult.pagination?.totalResults || 0} total matches`); - logger.debug(`${FILE_NAME}: About to generate blocks for search results`); - + if (!searchResult.success) { logger.error(`${FILE_NAME}: Search failed: ${searchResult.message}`); await respond({ @@ -84,18 +82,17 @@ const handleCommand = async (command, respond) => { }); return; } - + // Get total count for validation const totalCount = searchResult.pagination?.totalResults || 0; - + // Check if search returned too many results if (totalCount > MAX_RESULTS_THRESHOLD) { logger.warn(`${FILE_NAME}: Search for "${keyword}" returned too many results (${totalCount}), displaying first page with warning`); - // Continue processing but add a notification searchResult.tooManyResults = true; } - + if (!searchResult.results || searchResult.results.length === 0) { if (totalCount > 0) { logger.warn(`${FILE_NAME}: No rules found on page ${page} for "${keyword}", but ${totalCount} total matches exist`); @@ -112,16 +109,14 @@ const handleCommand = async (command, respond) => { } return; } - + // Generate blocks with pagination support let blocks; try { logger.debug(`${FILE_NAME}: Calling getSearchResultBlocks with ${searchResult.results.length} results`); - // If we have too many results, add a warning block at the beginning if (searchResult.tooManyResults) { blocks = getSearchResultBlocks(keyword, searchResult.results, searchResult.pagination); - // Insert warning at the beginning of blocks (after the header) blocks.splice(1, 0, { "type": "section", @@ -133,7 +128,6 @@ const handleCommand = async (command, respond) => { } else { blocks = getSearchResultBlocks(keyword, searchResult.results, searchResult.pagination); } - logger.debug(`${FILE_NAME}: Successfully generated ${blocks?.length || 0} blocks`); } catch (blockError) { // Use error handler for block generation errors @@ -143,19 +137,19 @@ const handleCommand = async (command, respond) => { }); return; } - + // Add debug log before sending response logger.debug(`${FILE_NAME}: About to send response with ${blocks?.length || 0} blocks`); - + // Determine if this should be visible to everyone or just the user const isEphemeral = totalCount > 20; - + // Respond with the search results await respond({ blocks: blocks, response_type: isEphemeral ? 'ephemeral' : 'in_channel' }); - + // Add debug log after sending response logger.debug(`${FILE_NAME}: Response sent successfully`); } catch (error) { @@ -166,6 +160,128 @@ const handleCommand = async (command, respond) => { } }; +/** + * Handle the complex search command for Sigma rules + * Processes advanced search queries with multiple conditions + * + * @param {Object} command - The Slack command object + * @param {Function} respond - Function to send response back to Slack + */ +const handleComplexSearch = async (command, respond) => { + try { + logger.debug(`${FILE_NAME}: Processing complex search command: ${JSON.stringify(command.text)}`); + + if (!command || !command.text) { + logger.warn(`${FILE_NAME}: Empty command received for complex search`); + await respond('Invalid command. Usage: /sigma-search where [conditions]'); + return; + } + + // Extract query string + let queryString = command.text.trim(); + let page = 1; + let pageSize = MAX_RESULTS_PER_PAGE; + + // Check for pagination format: query page=X + const pagingMatch = queryString.match(/(.+)\s+page=(\d+)$/i); + if (pagingMatch) { + queryString = pagingMatch[1].trim(); + page = parseInt(pagingMatch[2], 10) || 1; + logger.debug(`${FILE_NAME}: Detected pagination request in complex search: page ${page}`); + } + + // Check for page size format: query limit=X + const limitMatch = queryString.match(/(.+)\s+limit=(\d+)$/i); + if (limitMatch) { + queryString = limitMatch[1].trim(); + pageSize = parseInt(limitMatch[2], 10) || MAX_RESULTS_PER_PAGE; + // Ensure the page size is within reasonable limits + pageSize = Math.min(Math.max(pageSize, 1), 100); + logger.debug(`${FILE_NAME}: Detected page size request in complex search: limit ${pageSize}`); + } + + logger.info(`${FILE_NAME}: Performing complex search with query: ${queryString}`); + + await respond({ + text: 'Processing complex search query... This may take a moment.', + response_type: 'ephemeral' + }); + + // Perform the complex search + const searchResult = await searchSigmaRulesComplex(queryString, page, pageSize); + + if (!searchResult.success) { + logger.error(`${FILE_NAME}: Complex search failed: ${searchResult.message}`); + await respond({ + text: `Search failed: ${searchResult.message}`, + response_type: 'ephemeral' + }); + return; + } + + // Check if we have results + if (!searchResult.results || searchResult.results.length === 0) { + logger.warn(`${FILE_NAME}: No rules found matching complex query criteria`); + await respond({ + text: `No rules found matching the specified criteria.`, + response_type: 'ephemeral' + }); + return; + } + + // Generate blocks with pagination support + let blocks; + try { + // Use the standard search result blocks but with a modified header + blocks = getSearchResultBlocks( + `Complex Query: ${queryString}`, + searchResult.results, + searchResult.pagination + ); + + // Replace the header to indicate it's a complex search + if (blocks && blocks.length > 0) { + blocks[0] = { + type: "header", + text: { + type: "plain_text", + text: `Sigma Rule Search Results - Query`, + emoji: true + } + }; + + // Add a description of the search criteria + blocks.splice(1, 0, { + type: "section", + text: { + type: "mrkdwn", + text: `*Query:* \`${queryString}\`` + } + }); + } + } catch (blockError) { + await handleError(blockError, `${FILE_NAME}: Complex search block generation`, respond, { + responseType: 'ephemeral', + customMessage: `Error generating results view: ${blockError.message}` + }); + return; + } + + // Respond with the search results + await respond({ + blocks: blocks, + response_type: 'ephemeral' // Complex searches are usually more specific to the user + }); + + logger.info(`${FILE_NAME}: Complex search response sent successfully with ${searchResult.results.length} results`); + } catch (error) { + await handleError(error, `${FILE_NAME}: Complex search handler`, respond, { + responseType: 'ephemeral' + }); + } +}; + module.exports = { - handleCommand + handleCommand, + handleComplexSearch }; \ No newline at end of file diff --git a/src/lang/command_patterns.js b/src/lang/command_patterns.js index 22f3e98..bcd38f3 100644 --- a/src/lang/command_patterns.js +++ b/src/lang/command_patterns.js @@ -30,21 +30,14 @@ const commandPatterns = [ module: 'sigma', params: [2] // rule ID is in capturing group 2 }, - + // Sigma search patterns { name: 'sigma-search', - regex: /^(search|find|look\s+for)\s+(rules|detections)?\s*(in|from)?\s*sigma\s+(for|where|with)?\s+(.+)$/i, - action: 'search', + regex: /^(search|find)\s+(sigma\s+)?(rules|detections)?\s*(where|with)\s+(.+)$/i, + action: 'complexSearch', module: 'sigma', - params: [5] // search query is in capturing group 5 - }, - { - name: 'sigma-search-simple', - regex: /^(search|find)\s+(.+)$/i, - action: 'search', - module: 'sigma', - params: [2] // search query is in capturing group 2 + params: [5] // complex query conditions in capturing group 5 }, // Sigma create patterns @@ -55,7 +48,7 @@ const commandPatterns = [ module: 'sigma', params: [2] // rule ID is in capturing group 2 }, - + // Sigma stats patterns { name: 'sigma-stats-first', diff --git a/src/lang/query_parser.js b/src/lang/query_parser.js new file mode 100644 index 0000000..0f949cc --- /dev/null +++ b/src/lang/query_parser.js @@ -0,0 +1,197 @@ +/** + * query_parser.js + * + * Utility to parse complex search queries for Sigma rules + * Handles conditions like title contains "X", tags include "Y", etc. + */ + +const logger = require('../utils/logger'); +const FILE_NAME = 'query_parser.js'; + +/** + * Parse a complex query string into structured search parameters + * Supports conditions like: + * - title contains "ransomware" + * - logsource.category == "process_creation" + * - tags include privilege_escalation + * - modified after 2024-01-01 + * - author is "John Doe" + * + * Also supports logical operators: + * - AND, and + * - OR, or + * + * @param {string} queryString - The complex query string to parse + * @returns {Object} Structured search parameters + */ +function parseComplexQuery(queryString) { + try { + logger.debug(`${FILE_NAME}: Parsing complex query: ${queryString}`); + + if (!queryString || typeof queryString !== 'string') { + logger.warn(`${FILE_NAME}: Invalid query string`); + return { valid: false, error: 'Invalid query string' }; + } + + // Initialize the result object + const result = { + valid: true, + conditions: [], + operator: 'AND' // Default to AND for multiple conditions + }; + + // Check for explicit logical operators + if (/ AND /i.test(queryString)) { + result.operator = 'AND'; + // Split by AND and parse each part + const parts = queryString.split(/ AND /i); + for (const part of parts) { + const condition = parseCondition(part.trim()); + if (condition) { + result.conditions.push(condition); + } + } + } else if (/ OR /i.test(queryString)) { + result.operator = 'OR'; + // Split by OR and parse each part + const parts = queryString.split(/ OR /i); + for (const part of parts) { + const condition = parseCondition(part.trim()); + if (condition) { + result.conditions.push(condition); + } + } + } else { + // Single condition + const condition = parseCondition(queryString.trim()); + if (condition) { + result.conditions.push(condition); + } + } + + // If no valid conditions found, mark as invalid + if (result.conditions.length === 0) { + result.valid = false; + result.error = 'No valid search conditions found'; + } + + logger.debug(`${FILE_NAME}: Parsed query result: ${JSON.stringify(result)}`); + return result; + } catch (error) { + logger.error(`${FILE_NAME}: Error parsing complex query: ${error.message}`); + return { + valid: false, + error: `Error parsing query: ${error.message}` + }; + } +} + +/** + * Parse a single condition from the query string + * + * @param {string} conditionStr - The condition string to parse + * @returns {Object|null} Parsed condition object or null if invalid + */ +function parseCondition(conditionStr) { + logger.debug(`${FILE_NAME}: Parsing condition: ${conditionStr}`); + + // Define regex patterns for different condition types + const patterns = [ + // title contains "value" + { + regex: /^(title|name)\s+(contains|has|like|includes)\s+"?([^"]+)"?$/i, + handler: (matches) => ({ + field: 'title', + operator: 'contains', + value: matches[3].trim() + }) + }, + // description contains "value" + { + regex: /^(description|desc)\s+(contains|has|like|includes)\s+"?([^"]+)"?$/i, + handler: (matches) => ({ + field: 'description', + operator: 'contains', + value: matches[3].trim() + }) + }, + // logsource.category == "value" or logsource.category = "value" + { + regex: /^logsource\.(\w+)\s*(==|=|equals?)\s*"?([^"]+)"?$/i, + handler: (matches) => ({ + field: 'logsource', + subfield: matches[1].toLowerCase(), + operator: 'equals', + value: matches[3].trim() + }) + }, + // tags include "value" or tag contains "value" + { + regex: /^tags?\s+(includes?|contains|has)\s+"?([^"]+)"?$/i, + handler: (matches) => ({ + field: 'tags', + operator: 'contains', + value: matches[2].trim() + }) + }, + // modified after YYYY-MM-DD + { + regex: /^(modified|updated|created|date)\s+(after|before|on|since)\s+"?(\d{4}-\d{2}-\d{2})"?$/i, + handler: (matches) => ({ + field: 'date', + type: matches[1].toLowerCase(), + operator: matches[2].toLowerCase(), + value: matches[3].trim() + }) + }, + // author is "value" or author = "value" + { + regex: /^(author|creator)\s+(is|equals?|==|=)\s+"?([^"]+)"?$/i, + handler: (matches) => ({ + field: 'author', + operator: 'equals', + value: matches[3].trim() + }) + }, + // level is "value" or level = "value" + { + regex: /^(level|severity)\s+(is|equals?|==|=)\s+"?([^"]+)"?$/i, + handler: (matches) => ({ + field: 'level', + operator: 'equals', + value: matches[3].trim() + }) + }, + // id is "value" or id = "value" + { + regex: /^(id|identifier)\s+(is|equals?|==|=)\s+"?([^"]+)"?$/i, + handler: (matches) => ({ + field: 'id', + operator: 'equals', + value: matches[3].trim() + }) + } + ]; + + // Try each pattern + for (const pattern of patterns) { + const matches = conditionStr.match(pattern.regex); + if (matches) { + return pattern.handler(matches); + } + } + + // If we get here, no patterns matched + logger.warn(`${FILE_NAME}: No pattern matched condition: ${conditionStr}`); + + // Default to simple keyword search if no specific pattern matches + return { + field: 'keyword', + operator: 'contains', + value: conditionStr.trim() + }; +} + +module.exports = { + parseComplexQuery +}; \ No newline at end of file diff --git a/src/services/sigma/sigma_search_service.js b/src/services/sigma/sigma_search_service.js index 81b53dc..4d17ad3 100644 --- a/src/services/sigma/sigma_search_service.js +++ b/src/services/sigma/sigma_search_service.js @@ -1,14 +1,15 @@ /** * sigma_search_service.js * - * This service provides functionality for searching Sigma rules by keywords. + * This service provides functionality for searching Sigma rules by keywords and complex queries. * It processes search results and returns them in a structured format. * Supports pagination for large result sets. */ -const { searchRules } = require('../../sigma_db/sigma_db_queries'); + +const { searchRules, searchRulesComplex } = require('../../sigma_db/sigma_db_queries'); +const { parseComplexQuery } = require('../../lang/query_parser'); const logger = require('../../utils/logger'); const { convertSigmaRule } = require('./sigma_converter_service'); - const { getFileName } = require('../../utils/file_utils'); const FILE_NAME = getFileName(__filename); @@ -153,6 +154,109 @@ async function searchSigmaRules(keyword, page = 1, pageSize = 10) { } } +/** + * Searches for Sigma rules using complex query conditions + * Supports filtering by title, logsource, tags, dates, and more + * + * @param {string} queryString - The complex query string to parse + * @param {number} page - Page number (1-based index, default: 1) + * @param {number} pageSize - Number of results per page (default: 10) + * @returns {Promise} Result object with success flag and processed results + */ +async function searchSigmaRulesComplex(queryString, page = 1, pageSize = 10) { + if (!queryString || typeof queryString !== 'string') { + logger.warn(`${FILE_NAME}: Cannot perform complex search: Missing or invalid query string`); + return { + success: false, + message: 'Missing or invalid complex query' + }; + } + + // Validate pagination parameters + if (typeof page !== 'number' || page < 1) { + logger.warn(`${FILE_NAME}: Invalid page number: ${page}, defaulting to 1`); + page = 1; + } + + if (typeof pageSize !== 'number' || pageSize < 1 || pageSize > 100) { + logger.warn(`${FILE_NAME}: Invalid page size: ${pageSize}, defaulting to 10`); + pageSize = 10; + } + + // Calculate the offset based on page number + const offset = (page - 1) * pageSize; + + logger.info(`${FILE_NAME}: Performing complex search with query: "${queryString}" (page ${page}, size ${pageSize})`); + + try { + // Parse the complex query string + const parsedQuery = parseComplexQuery(queryString); + + if (!parsedQuery.valid) { + logger.warn(`${FILE_NAME}: Invalid complex query: ${parsedQuery.error}`); + return { + success: false, + message: `Invalid query: ${parsedQuery.error}` + }; + } + + // Perform the database search with the parsed query + const searchResult = await searchRulesComplex(parsedQuery, pageSize, offset); + + // Defensive handling of possible return formats + let allResults = []; + let totalCount = 0; + + // Handle search results + if (searchResult) { + if (Array.isArray(searchResult.results)) { + allResults = searchResult.results; + totalCount = searchResult.totalCount || 0; + } + } + + if (allResults.length === 0) { + return { + success: true, + results: [], + message: `No rules found matching the complex query criteria`, + pagination: { + currentPage: page, + pageSize: pageSize, + totalPages: Math.ceil(totalCount / pageSize), + totalResults: totalCount, + hasMore: false + } + }; + } + + // Calculate pagination info + const totalPages = Math.ceil(totalCount / pageSize); + const hasMore = (offset + pageSize) < totalCount; + + return { + success: true, + results: allResults, + count: allResults.length, + query: parsedQuery, + pagination: { + currentPage: page, + pageSize: pageSize, + totalPages: totalPages, + totalResults: totalCount, + hasMore: hasMore + } + }; + } catch (error) { + logger.error(`${FILE_NAME}: Error in complex search: ${error.message}`); + logger.debug(`${FILE_NAME}: Error stack: ${error.stack}`); + return { + success: false, + message: `Error performing complex search: ${error.message}` + }; + } +} + /** * Enhanced search that returns fully converted rule objects with pagination support * This is a more expensive operation than basic search @@ -210,5 +314,6 @@ async function searchAndConvertRules(keyword, page = 1, pageSize = 10) { module.exports = { searchSigmaRules, + searchSigmaRulesComplex, searchAndConvertRules }; \ No newline at end of file diff --git a/src/sigma_db/sigma_db_initialize.js b/src/sigma_db/sigma_db_initialize.js index 934e7d5..c5342d5 100644 --- a/src/sigma_db/sigma_db_initialize.js +++ b/src/sigma_db/sigma_db_initialize.js @@ -58,41 +58,49 @@ async function initializeDatabase(db) { return; } - // Create rules table with basic information - const createRulesTableSql = ` - CREATE TABLE sigma_rules ( - id TEXT PRIMARY KEY, - file_path TEXT, - content TEXT, - date DATETIME DEFAULT CURRENT_TIMESTAMP - ) - `; - - db.run(createRulesTableSql, (err) => { + // Drop FTS table if exists + db.run('DROP TABLE IF EXISTS rule_search', (err) => { if (err) { reject(err); return; } - // Create rule_parameters table for individual parameters - const createParamsTableSql = ` - CREATE TABLE rule_parameters ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - rule_id TEXT, - param_name TEXT, - param_value TEXT, - param_type TEXT, - FOREIGN KEY (rule_id) REFERENCES sigma_rules(id) ON DELETE CASCADE + // Create rules table with basic information + const createRulesTableSql = ` + CREATE TABLE sigma_rules ( + id TEXT PRIMARY KEY, + file_path TEXT, + content TEXT, + date DATETIME DEFAULT CURRENT_TIMESTAMP ) `; - db.run(createParamsTableSql, (err) => { + db.run(createRulesTableSql, (err) => { if (err) { reject(err); - } else { - logger.info(`${FILE_NAME}: Database schema initialized`); - resolve(); + return; } + + // Create rule_parameters table for individual parameters + const createParamsTableSql = ` + CREATE TABLE rule_parameters ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + rule_id TEXT, + param_name TEXT, + param_value TEXT, + param_type TEXT, + FOREIGN KEY (rule_id) REFERENCES sigma_rules(id) ON DELETE CASCADE + ) + `; + + db.run(createParamsTableSql, (err) => { + if (err) { + reject(err); + } else { + logger.info(`${FILE_NAME}: Database schema initialized`); + resolve(); + } + }); }); }); }); @@ -100,6 +108,70 @@ async function initializeDatabase(db) { }); } +// Create FTS5 virtual table for full-text search +async function createFtsTable(db) { + return new Promise((resolve, reject) => { + logger.info(`${FILE_NAME}: Creating FTS5 virtual table for full-text search`); + + // Create the FTS5 virtual table + const createFtsTableSql = ` + CREATE VIRTUAL TABLE IF NOT EXISTS rule_search USING fts5( + rule_id, + title, + description, + logsource, + tags, + author, + level, + content, + tokenize="unicode61" + ); + `; + + db.run(createFtsTableSql, (err) => { + if (err) { + logger.error(`${FILE_NAME}: Failed to create FTS5 table: ${err.message}`); + reject(err); + } else { + logger.info(`${FILE_NAME}: FTS5 virtual table created successfully`); + resolve(); + } + }); + }); +} + +// Populate FTS table with rule data for full-text search +async function populateFtsTable(db) { + return new Promise((resolve, reject) => { + logger.info(`${FILE_NAME}: Populating FTS5 table with rule data`); + + // Insert query that aggregates data from both tables + const populateFtsSql = ` + INSERT INTO rule_search(rule_id, title, description, logsource, tags, author, level, content) + SELECT + r.id, + (SELECT param_value FROM rule_parameters WHERE rule_id = r.id AND param_name = 'title' LIMIT 1), + (SELECT param_value FROM rule_parameters WHERE rule_id = r.id AND param_name = 'description' LIMIT 1), + (SELECT param_value FROM rule_parameters WHERE rule_id = r.id AND param_name = 'logsource' LIMIT 1), + (SELECT param_value FROM rule_parameters WHERE rule_id = r.id AND param_name = 'tags' LIMIT 1), + (SELECT param_value FROM rule_parameters WHERE rule_id = r.id AND param_name = 'author' LIMIT 1), + (SELECT param_value FROM rule_parameters WHERE rule_id = r.id AND param_name = 'level' LIMIT 1), + r.content + FROM sigma_rules r + `; + + db.run(populateFtsSql, (err) => { + if (err) { + logger.error(`${FILE_NAME}: Failed to populate FTS5 table: ${err.message}`); + reject(err); + } else { + logger.info(`${FILE_NAME}: FTS5 table populated successfully`); + resolve(); + } + }); + }); +} + // Determine if a YAML document is a Sigma rule function isSigmaRule(doc) { // Check for essential Sigma rule properties @@ -531,6 +603,12 @@ async function main() { // Create indexes await createIndexes(db); + // Create FTS5 table + await createFtsTable(db); + + // Populate FTS5 table with rule data + await populateFtsTable(db); + // Close database connection db.close((err) => { if (err) { @@ -556,5 +634,7 @@ if (require.main === module) { module.exports = { initializeDatabase, importRules, - createIndexes + createIndexes, + createFtsTable, + populateFtsTable }; \ No newline at end of file diff --git a/src/sigma_db/sigma_db_queries.js b/src/sigma_db/sigma_db_queries.js index 1867c79..8fa53f9 100644 --- a/src/sigma_db/sigma_db_queries.js +++ b/src/sigma_db/sigma_db_queries.js @@ -198,6 +198,17 @@ async function searchRules(keyword, limit = 10, offset = 0) { db = await getDbConnection(); logger.debug(`${FILE_NAME}: Database connection established for search`); + // Use FTS5 for faster searching if available + const ftsAvailable = await checkFtsAvailable(db); + + if (ftsAvailable) { + logger.debug(`${FILE_NAME}: Using FTS5 for keyword search`); + return searchRulesFTS(keyword, limit, offset); + } + + // If FTS5 is not available, use the legacy search method + logger.debug(`${FILE_NAME}: FTS5 not available, using legacy search method`); + // First get the total count of matching rules (for pagination info) const countQuery = ` SELECT COUNT(*) as count @@ -264,6 +275,613 @@ async function searchRules(keyword, limit = 10, offset = 0) { } } +/** + * Check if FTS5 virtual table is available + * + * @param {Object} db - Database connection + * @returns {Promise} Whether FTS5 is available + */ +async function checkFtsAvailable(db) { + try { + const result = await new Promise((resolve, reject) => { + db.get("SELECT name FROM sqlite_master WHERE type='table' AND name='rule_search'", (err, row) => { + if (err) { + logger.error(`${FILE_NAME}: Error checking for FTS5 table: ${err.message}`); + reject(err); + } else { + resolve(row !== undefined); + } + }); + }); + + logger.debug(`${FILE_NAME}: FTS5 table availability check: ${result ? 'Available' : 'Not available'}`); + return result; + } catch (error) { + logger.error(`${FILE_NAME}: Error checking FTS availability: ${error.message}`); + return false; + } +} + +/** + * Search for Sigma rules using FTS5 + * Performs a full-text search and returns matching rules with pagination + * + * @param {string} keyword - The keyword to search for + * @param {number} limit - Maximum number of results to return (default: 10) + * @param {number} offset - Number of results to skip (for pagination, default: 0) + * @returns {Promise} Object with results array and total count + */ +async function searchRulesFTS(keyword, limit = 10, offset = 0) { + if (!keyword) { + logger.warn(`${FILE_NAME}: Empty search keyword provided for FTS search`); + return { results: [], totalCount: 0 }; + } + + // Prepare FTS query - add * for prefix matching if not already present + let ftsQuery = keyword.trim(); + if (!ftsQuery.endsWith('*')) { + ftsQuery = `${ftsQuery}*`; + } + + logger.info(`${FILE_NAME}: Performing FTS search with query: "${ftsQuery}" (limit: ${limit}, offset: ${offset})`); + + let db; + try { + db = await getDbConnection(); + logger.debug(`${FILE_NAME}: Database connection established for FTS search`); + + // First get the total count of matching rules + const countQuery = ` + SELECT COUNT(*) as count + FROM rule_search + WHERE rule_search MATCH ? + `; + + const countResult = await new Promise((resolve, reject) => { + db.get(countQuery, [ftsQuery], (err, row) => { + if (err) { + logger.error(`${FILE_NAME}: FTS count query error: ${err.message}`); + reject(err); + } else { + resolve(row || { count: 0 }); + } + }); + }); + + const totalCount = countResult.count; + logger.debug(`${FILE_NAME}: Total matching rules for FTS query "${ftsQuery}": ${totalCount}`); + + // Now get the actual results with pagination + const searchQuery = ` + SELECT rule_id, title + FROM rule_search + WHERE rule_search MATCH ? + ORDER BY rank + LIMIT ? OFFSET ? + `; + + const results = await new Promise((resolve, reject) => { + db.all(searchQuery, [ftsQuery, limit, offset], (err, rows) => { + if (err) { + logger.error(`${FILE_NAME}: FTS search query error: ${err.message}`); + reject(err); + } else { + logger.debug(`${FILE_NAME}: FTS search query returned ${rows ? rows.length : 0} results`); + resolve(rows || []); + } + }); + }); + + logger.debug(`${FILE_NAME}: FTS search results page for query "${ftsQuery}": ${results.length} matches (page ${Math.floor(offset / limit) + 1})`); + + return { + results: results.map(r => ({ id: r.rule_id, title: r.title || r.rule_id })), + totalCount + }; + } catch (error) { + logger.error(`${FILE_NAME}: Error in FTS search operation: ${error.message}`); + logger.debug(`${FILE_NAME}: FTS search error stack: ${error.stack}`); + return { results: [], totalCount: 0 }; + } finally { + if (db) { + try { + await db.close(); + logger.debug(`${FILE_NAME}: Database connection closed after FTS search operation`); + } catch (closeError) { + logger.error(`${FILE_NAME}: Error closing database connection after FTS search: ${closeError.message}`); + } + } + } +} + +/** + * Search for Sigma rules using complex query conditions + * Supports filtering by multiple attributes like title, logsource, tags, etc. + * + * @param {Object} parsedQuery - The parsed query object containing conditions and operator + * @param {number} limit - Maximum number of results to return + * @param {number} offset - Number of results to skip (for pagination) + * @returns {Promise} Object with results array and total count + */ +async function searchRulesComplex(parsedQuery, limit = 10, offset = 0) { + if (!parsedQuery || !parsedQuery.valid) { + logger.warn(`${FILE_NAME}: Invalid query object provided`); + return { results: [], totalCount: 0 }; + } + + logger.info(`${FILE_NAME}: Performing complex search with ${parsedQuery.conditions.length} conditions (limit: ${limit}, offset: ${offset})`); + + let db; + // Declare this at function scope so it's available in the finally block + let usingFts = false; + + try { + db = await getDbConnection(); + logger.debug(`${FILE_NAME}: Database connection established for complex search`); + + // Check if FTS5 is available + const ftsAvailable = await checkFtsAvailable(db); + + if (ftsAvailable) { + logger.debug(`${FILE_NAME}: Using FTS5 for complex search`); + // Set flag that we're using FTS + usingFts = true; + // Pass db connection to searchRulesComplexFTS and let that function manage it + const results = await searchRulesComplexFTS(parsedQuery, limit, offset, db); + return results; + } + + logger.debug(`${FILE_NAME}: FTS5 not available, using legacy complex search method`); + + // Build the SQL query based on the conditions + const { sqlQuery, sqlCountQuery, params } = buildComplexSqlQuery(parsedQuery, limit, offset); + + logger.debug(`${FILE_NAME}: Executing complex search SQL: ${sqlQuery}`); + logger.debug(`${FILE_NAME}: Query parameters: ${JSON.stringify(params)}`); + + // First get the total count of matching results + const countResult = await new Promise((resolve, reject) => { + db.get(sqlCountQuery, params.slice(0, params.length - 2), (err, row) => { + if (err) { + logger.error(`${FILE_NAME}: Complex search count query error: ${err.message}`); + reject(err); + } else { + resolve(row || { count: 0 }); + } + }); + }); + + const totalCount = countResult.count; + logger.debug(`${FILE_NAME}: Total matching rules for complex query: ${totalCount}`); + + // Now get the actual results with pagination + const results = await new Promise((resolve, reject) => { + db.all(sqlQuery, params, (err, rows) => { + if (err) { + logger.error(`${FILE_NAME}: Complex search query error: ${err.message}`); + reject(err); + } else { + logger.debug(`${FILE_NAME}: Complex search query returned ${rows ? rows.length : 0} results`); + resolve(rows || []); + } + }); + }); + + // Format the results + const formattedResults = results.map(r => ({ + id: r.rule_id, + title: r.title || r.rule_id + })); + + logger.debug(`${FILE_NAME}: Returning ${formattedResults.length} results for complex search`); + + return { + results: formattedResults, + totalCount + }; + } catch (error) { + logger.error(`${FILE_NAME}: Error in complex search operation: ${error.message}`); + logger.debug(`${FILE_NAME}: Complex search error stack: ${error.stack}`); + return { results: [], totalCount: 0 }; + } finally { + // IMPORTANT: Only close the db connection if we're not using FTS + // When using FTS, let searchRulesComplexFTS manage the connection + if (db && !usingFts) { + try { + await new Promise((resolve) => db.close(() => resolve())); + logger.debug(`${FILE_NAME}: Database connection closed after complex search operation`); + } catch (closeError) { + logger.error(`${FILE_NAME}: Error closing database after complex search: ${closeError.message}`); + } + } + } +} + +/** + * Search for Sigma rules using complex query conditions with FTS5 + * Uses the FTS5 virtual table for faster text searching + * + * @param {Object} parsedQuery - The parsed query object + * @param {number} limit - Maximum number of results to return + * @param {number} offset - Number of results to skip (for pagination) + * @param {Object} providedDb - Database connection (optional, will create one if not provided) + * @returns {Promise} Object with results array and total count + */ +async function searchRulesComplexFTS(parsedQuery, limit = 10, offset = 0, providedDb = null) { + if (!parsedQuery || !parsedQuery.valid) { + logger.warn(`${FILE_NAME}: Invalid query object provided for FTS complex search`); + return { results: [], totalCount: 0 }; + } + + logger.info(`${FILE_NAME}: Performing complex FTS search with ${parsedQuery.conditions.length} conditions`); + + let db; + let shouldCloseDb = false; + + try { + // Use provided db connection or create a new one + if (providedDb) { + db = providedDb; + } else { + db = await getDbConnection(); + shouldCloseDb = true; + logger.debug(`${FILE_NAME}: Created new database connection for complex FTS search`); + } + + // Build FTS query from conditions + const { ftsQuery, whereClause, params } = buildComplexFtsQuery(parsedQuery); + + logger.debug(`${FILE_NAME}: FTS query: "${ftsQuery}", additional where: ${whereClause ? whereClause : 'none'}`); + logger.debug(`${FILE_NAME}: Query parameters: ${JSON.stringify(params)}`); + + // Build count query + let countQuery; + let countParams; + + if (whereClause) { + countQuery = ` + SELECT COUNT(*) as count + FROM rule_search + WHERE rule_search MATCH ? + AND ${whereClause} + `; + countParams = [ftsQuery, ...params]; + } else { + countQuery = ` + SELECT COUNT(*) as count + FROM rule_search + WHERE rule_search MATCH ? + `; + countParams = [ftsQuery]; + } + + // Get total count + const countResult = await new Promise((resolve, reject) => { + db.get(countQuery, countParams, (err, row) => { + if (err) { + logger.error(`${FILE_NAME}: Complex FTS count query error: ${err.message}`); + reject(err); + } else { + resolve(row || { count: 0 }); + } + }); + }); + + const totalCount = countResult.count; + logger.debug(`${FILE_NAME}: Total matching rules for complex FTS query: ${totalCount}`); + + // Build results query with pagination + let searchQuery; + let searchParams; + + if (whereClause) { + searchQuery = ` + SELECT rule_id, title + FROM rule_search + WHERE rule_search MATCH ? + AND ${whereClause} + ORDER BY rank + LIMIT ? OFFSET ? + `; + searchParams = [ftsQuery, ...params, limit, offset]; + } else { + searchQuery = ` + SELECT rule_id, title + FROM rule_search + WHERE rule_search MATCH ? + ORDER BY rank + LIMIT ? OFFSET ? + `; + searchParams = [ftsQuery, limit, offset]; + } + + // Get paginated results + const results = await new Promise((resolve, reject) => { + db.all(searchQuery, searchParams, (err, rows) => { + if (err) { + logger.error(`${FILE_NAME}: Complex FTS search query error: ${err.message}`); + reject(err); + } else { + logger.debug(`${FILE_NAME}: Complex FTS search query returned ${rows ? rows.length : 0} results`); + resolve(rows || []); + } + }); + }); + + // Format the results + const formattedResults = results.map(r => ({ + id: r.rule_id, + title: r.title || r.rule_id + })); + + logger.debug(`${FILE_NAME}: Returning ${formattedResults.length} results for complex FTS search`); + + return { + results: formattedResults, + totalCount + }; + } catch (error) { + logger.error(`${FILE_NAME}: Error in complex FTS search operation: ${error.message}`); + logger.debug(`${FILE_NAME}: Complex FTS search error stack: ${error.stack}`); + return { results: [], totalCount: 0 }; + } finally { + // Only close the database if we created it AND we're not in the middle of a transaction + if (db && shouldCloseDb) { + try { + await db.close(); + logger.debug(`${FILE_NAME}: Database connection closed after complex FTS search`); + } catch (closeError) { + logger.error(`${FILE_NAME}: Error closing database after complex FTS search: ${closeError.message}`); + } + } + } +} + +/** + * Build FTS query and WHERE clause from parsed query conditions + * + * @param {Object} parsedQuery - The parsed query object + * @returns {Object} Object with FTS query, additional WHERE clause, and parameters + */ +function buildComplexFtsQuery(parsedQuery) { + const { conditions, operator } = parsedQuery; + + // Separate text search conditions from other conditions + const textConditions = []; + const nonTextConditions = []; + + for (const condition of conditions) { + switch (condition.field) { + case 'title': + case 'description': + case 'author': + case 'tags': + case 'keyword': + // These can be handled by FTS directly + textConditions.push(condition); + break; + default: + // These need additional WHERE clauses + nonTextConditions.push(condition); + break; + } + } + + // Build FTS MATCH query + let ftsQueryParts = []; + + for (const condition of textConditions) { + let fieldPrefix = ''; + + // Add field-specific prefix if available + if (condition.field !== 'keyword') { + fieldPrefix = `${condition.field}:`; + } + + // Add wildcard for partial matching if not already present + let value = condition.value.trim(); + if (!value.endsWith('*')) { + value = `${value}*`; + } + + ftsQueryParts.push(`${fieldPrefix}${value}`); + } + + // If no text conditions, use a match-all query + const ftsQuery = ftsQueryParts.length > 0 + ? ftsQueryParts.join(operator === 'AND' ? ' AND ' : ' OR ') + : '*'; + + // Build additional WHERE clauses for non-text conditions + let whereClauseParts = []; + const params = []; + + for (const condition of nonTextConditions) { + switch (condition.field) { + case 'date': + const dateOperator = condition.operator === 'after' ? '>' : + condition.operator === 'before' ? '<' : '='; + whereClauseParts.push(`date ${dateOperator} date(?)`); + params.push(condition.value); + break; + + case 'level': + whereClauseParts.push(`level = ?`); + params.push(condition.value); + break; + + case 'logsource': + whereClauseParts.push(`logsource LIKE ?`); + params.push(`%${condition.subfield}%${condition.value}%`); + break; + + case 'id': + whereClauseParts.push(`rule_id = ?`); + params.push(condition.value); + break; + } + } + + // Combine WHERE clauses + const whereClause = whereClauseParts.length > 0 + ? whereClauseParts.join(operator === 'AND' ? ' AND ' : ' OR ') + : ''; + + return { ftsQuery, whereClause, params }; +} + +/** + * Build the SQL query for complex search based on parsed conditions + * + * @param {Object} parsedQuery - The parsed query object + * @param {number} limit - Results limit + * @param {number} offset - Results offset + * @returns {Object} Object with SQL query, count query, and parameters + */ +function buildComplexSqlQuery(parsedQuery, limit, offset) { + const { conditions, operator } = parsedQuery; + const params = []; + + // Start building the primary table selection + let sqlSelectPart = ` + SELECT DISTINCT r.id as rule_id, + (SELECT param_value FROM rule_parameters WHERE rule_id = r.id AND param_name = 'title' LIMIT 1) as title + FROM sigma_rules r + `; + + // Build WHERE clause based on conditions + let whereClauses = []; + let joinIdx = 0; + + for (const condition of conditions) { + let whereClause = ''; + + switch (condition.field) { + case 'title': + joinIdx++; + whereClause = `EXISTS ( + SELECT 1 FROM rule_parameters p${joinIdx} + WHERE p${joinIdx}.rule_id = r.id + AND p${joinIdx}.param_name = 'title' + AND INSTR(LOWER(p${joinIdx}.param_value), LOWER(?)) > 0 + )`; + params.push(condition.value); + break; + + case 'description': + joinIdx++; + whereClause = `EXISTS ( + SELECT 1 FROM rule_parameters p${joinIdx} + WHERE p${joinIdx}.rule_id = r.id + AND p${joinIdx}.param_name = 'description' + AND INSTR(LOWER(p${joinIdx}.param_value), LOWER(?)) > 0 + )`; + params.push(condition.value); + break; + + case 'logsource': + joinIdx++; + whereClause = `EXISTS ( + SELECT 1 FROM rule_parameters p${joinIdx} + WHERE p${joinIdx}.rule_id = r.id + AND p${joinIdx}.param_name = 'logsource' + AND INSTR(LOWER(p${joinIdx}.param_value), LOWER('"${condition.subfield}":"${condition.value}"')) > 0 + )`; + break; + + case 'tags': + joinIdx++; + whereClause = `EXISTS ( + SELECT 1 FROM rule_parameters p${joinIdx} + WHERE p${joinIdx}.rule_id = r.id + AND p${joinIdx}.param_name = 'tags' + AND INSTR(LOWER(p${joinIdx}.param_value), LOWER(?)) > 0 + )`; + params.push(condition.value); + break; + + case 'date': + joinIdx++; + const dateOperator = condition.operator === 'after' ? '>' : + condition.operator === 'before' ? '<' : '='; + whereClause = `r.date ${dateOperator} date(?)`; + params.push(condition.value); + break; + + case 'author': + joinIdx++; + whereClause = `EXISTS ( + SELECT 1 FROM rule_parameters p${joinIdx} + WHERE p${joinIdx}.rule_id = r.id + AND p${joinIdx}.param_name = 'author' + AND INSTR(LOWER(p${joinIdx}.param_value), LOWER(?)) > 0 + )`; + params.push(condition.value); + break; + + case 'level': + joinIdx++; + whereClause = `EXISTS ( + SELECT 1 FROM rule_parameters p${joinIdx} + WHERE p${joinIdx}.rule_id = r.id + AND p${joinIdx}.param_name = 'level' + AND LOWER(p${joinIdx}.param_value) = LOWER(?) + )`; + params.push(condition.value); + break; + + case 'id': + whereClause = `LOWER(r.id) = LOWER(?)`; + params.push(condition.value); + break; + + case 'keyword': + default: + // Default to searching in title + joinIdx++; + whereClause = `EXISTS ( + SELECT 1 FROM rule_parameters p${joinIdx} + WHERE p${joinIdx}.rule_id = r.id + AND p${joinIdx}.param_name = 'title' + AND INSTR(LOWER(p${joinIdx}.param_value), LOWER(?)) > 0 + )`; + params.push(condition.value); + break; + } + + if (whereClause) { + whereClauses.push(whereClause); + } + } + + // Combine the WHERE clauses with the appropriate operator + let whereStatement = ''; + if (whereClauses.length > 0) { + const combiner = operator === 'AND' ? ' AND ' : ' OR '; + whereStatement = `WHERE ${whereClauses.join(combiner)}`; + } + + // Complete queries + const sqlQuery = ` + ${sqlSelectPart} + ${whereStatement} + ORDER BY rule_id + LIMIT ? OFFSET ? + `; + + const sqlCountQuery = ` + SELECT COUNT(DISTINCT r.id) as count + FROM sigma_rules r + ${whereStatement} + `; + + // Add pagination parameters + params.push(limit); + params.push(offset); + + return { sqlQuery, sqlCountQuery, params }; +} + /** * Debug function to retrieve detailed information about a rule's content * Useful for diagnosing issues with rule retrieval and content parsing @@ -579,7 +1197,11 @@ module.exports = { getAllRuleIds, findRuleById, searchRules, + searchRulesFTS, + searchRulesComplex, + searchRulesComplexFTS, debugRuleContent, getRuleYamlContent, - getStatsFromDatabase + getStatsFromDatabase, + checkFtsAvailable }; \ No newline at end of file -- 2.39.5 From 85bb8958b8139163721f584de44abe8f5f6e2bf1 Mon Sep 17 00:00:00 2001 From: Charlotte Croce Date: Fri, 18 Apr 2025 15:45:26 -0400 Subject: [PATCH 07/15] refactor sigma_db_queries into multiple files --- .vscode/settings.json | 2 + src/services/sigma/sigma_backend_converter.js | 2 +- src/services/sigma/sigma_converter_service.js | 2 +- src/services/sigma/sigma_details_service.js | 4 +- src/services/sigma/sigma_search_service.js | 2 +- src/services/sigma/sigma_stats_service.js | 2 +- src/sigma_db/queries/complex-search.js | 259 ++++ src/sigma_db/queries/fts-search.js | 133 ++ src/sigma_db/queries/index.js | 34 + src/sigma_db/queries/query-builders.js | 258 ++++ src/sigma_db/queries/rule-retrieval.js | 164 +++ src/sigma_db/queries/simple-search.js | 118 ++ src/sigma_db/queries/stats-debug.js | 328 +++++ src/sigma_db/sigma_db_queries.js | 1207 ----------------- 14 files changed, 1302 insertions(+), 1213 deletions(-) create mode 100644 .vscode/settings.json create mode 100644 src/sigma_db/queries/complex-search.js create mode 100644 src/sigma_db/queries/fts-search.js create mode 100644 src/sigma_db/queries/index.js create mode 100644 src/sigma_db/queries/query-builders.js create mode 100644 src/sigma_db/queries/rule-retrieval.js create mode 100644 src/sigma_db/queries/simple-search.js create mode 100644 src/sigma_db/queries/stats-debug.js delete mode 100644 src/sigma_db/sigma_db_queries.js diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..7a73a41 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,2 @@ +{ +} \ No newline at end of file diff --git a/src/services/sigma/sigma_backend_converter.js b/src/services/sigma/sigma_backend_converter.js index b0330c4..59f63e5 100644 --- a/src/services/sigma/sigma_backend_converter.js +++ b/src/services/sigma/sigma_backend_converter.js @@ -11,7 +11,7 @@ const { execSync } = require('child_process'); const logger = require('../../utils/logger'); const { SIGMA_CLI_PATH, SIGMA_CLI_CONFIG } = require('../../config/appConfig'); const { convertSigmaRule } = require('./sigma_converter_service'); -const { getRuleYamlContent } = require('../../sigma_db/sigma_db_queries'); +const { getRuleYamlContent } = require('../../sigma_db/queries'); const { getFileName } = require('../../utils/file_utils'); const FILE_NAME = getFileName(__filename); diff --git a/src/services/sigma/sigma_converter_service.js b/src/services/sigma/sigma_converter_service.js index 5ae5679..1f57a17 100644 --- a/src/services/sigma/sigma_converter_service.js +++ b/src/services/sigma/sigma_converter_service.js @@ -4,7 +4,7 @@ // const logger = require('../../utils/logger'); const yaml = require('js-yaml'); -const { findRuleById } = require('../../sigma_db/sigma_db_queries'); +const { findRuleById } = require('../../sigma_db/queries'); const { getFileName } = require('../../utils/file_utils'); const FILE_NAME = getFileName(__filename); diff --git a/src/services/sigma/sigma_details_service.js b/src/services/sigma/sigma_details_service.js index 2f43019..e0dd364 100644 --- a/src/services/sigma/sigma_details_service.js +++ b/src/services/sigma/sigma_details_service.js @@ -5,7 +5,7 @@ */ const logger = require('../../utils/logger'); const { convertSigmaRule, extractDetectionCondition } = require('./sigma_converter_service'); -const { debugRuleContent, getRuleYamlContent } = require('../../sigma_db/sigma_db_queries'); +const { debugRuleContent, getRuleYamlContent } = require('../../sigma_db/queries'); const { getFileName } = require('../../utils/file_utils'); const FILE_NAME = getFileName(__filename); @@ -147,4 +147,4 @@ async function getSigmaRuleYaml(ruleId) { module.exports = { explainSigmaRule, getSigmaRuleYaml -}; +}; \ No newline at end of file diff --git a/src/services/sigma/sigma_search_service.js b/src/services/sigma/sigma_search_service.js index 4d17ad3..87b44f2 100644 --- a/src/services/sigma/sigma_search_service.js +++ b/src/services/sigma/sigma_search_service.js @@ -6,7 +6,7 @@ * Supports pagination for large result sets. */ -const { searchRules, searchRulesComplex } = require('../../sigma_db/sigma_db_queries'); +const { searchRules, searchRulesComplex } = require('../../sigma_db/queries'); const { parseComplexQuery } = require('../../lang/query_parser'); const logger = require('../../utils/logger'); const { convertSigmaRule } = require('./sigma_converter_service'); diff --git a/src/services/sigma/sigma_stats_service.js b/src/services/sigma/sigma_stats_service.js index c6ac0d6..0bee7a0 100644 --- a/src/services/sigma/sigma_stats_service.js +++ b/src/services/sigma/sigma_stats_service.js @@ -5,7 +5,7 @@ * Provides aggregated statistical information about the rule database */ const logger = require('../../utils/logger'); -const { getStatsFromDatabase } = require('../../sigma_db/sigma_db_queries'); +const { getStatsFromDatabase } = require('../../sigma_db/queries'); const { getFileName } = require('../../utils/file_utils'); const FILE_NAME = getFileName(__filename); diff --git a/src/sigma_db/queries/complex-search.js b/src/sigma_db/queries/complex-search.js new file mode 100644 index 0000000..8767e98 --- /dev/null +++ b/src/sigma_db/queries/complex-search.js @@ -0,0 +1,259 @@ +/** + * complex-search.js + * Functions for complex searching of Sigma rules + */ + +const { getDbConnection } = require('../sigma_db_connection'); +const logger = require('../../utils/logger'); +const { getFileName } = require('../../utils/file_utils'); +const { checkFtsAvailable } = require('./fts-search'); +const { buildComplexSqlQuery, buildComplexFtsQuery } = require('./query-builders'); +const FILE_NAME = getFileName(__filename); + +/** + * Search for Sigma rules using complex query conditions + * Supports filtering by multiple attributes like title, logsource, tags, etc. + * + * @param {Object} parsedQuery - The parsed query object containing conditions and operator + * @param {number} limit - Maximum number of results to return + * @param {number} offset - Number of results to skip (for pagination) + * @returns {Promise} Object with results array and total count + */ +async function searchRulesComplex(parsedQuery, limit = 10, offset = 0) { + if (!parsedQuery || !parsedQuery.valid) { + logger.warn(`${FILE_NAME}: Invalid query object provided`); + return { results: [], totalCount: 0 }; + } + + logger.info(`${FILE_NAME}: Performing complex search with ${parsedQuery.conditions.length} conditions (limit: ${limit}, offset: ${offset})`); + + let db; + // Declare this at function scope so it's available in the finally block + let usingFts = false; + + try { + db = await getDbConnection(); + logger.debug(`${FILE_NAME}: Database connection established for complex search`); + + // Check if FTS5 is available + const ftsAvailable = await checkFtsAvailable(db); + + if (ftsAvailable) { + logger.debug(`${FILE_NAME}: Using FTS5 for complex search`); + // Set flag that we're using FTS + usingFts = true; + // Pass db connection to searchRulesComplexFTS and let that function manage it + const results = await searchRulesComplexFTS(parsedQuery, limit, offset, db); + return results; + } + + logger.debug(`${FILE_NAME}: FTS5 not available, using legacy complex search method`); + + // Build the SQL query based on the conditions + const { sqlQuery, sqlCountQuery, params } = buildComplexSqlQuery(parsedQuery, limit, offset); + + logger.debug(`${FILE_NAME}: Executing complex search SQL: ${sqlQuery}`); + logger.debug(`${FILE_NAME}: Query parameters: ${JSON.stringify(params)}`); + + // First get the total count of matching results + const countResult = await new Promise((resolve, reject) => { + db.get(sqlCountQuery, params.slice(0, params.length - 2), (err, row) => { + if (err) { + logger.error(`${FILE_NAME}: Complex search count query error: ${err.message}`); + reject(err); + } else { + resolve(row || { count: 0 }); + } + }); + }); + + const totalCount = countResult.count; + logger.debug(`${FILE_NAME}: Total matching rules for complex query: ${totalCount}`); + + // Now get the actual results with pagination + const results = await new Promise((resolve, reject) => { + db.all(sqlQuery, params, (err, rows) => { + if (err) { + logger.error(`${FILE_NAME}: Complex search query error: ${err.message}`); + reject(err); + } else { + logger.debug(`${FILE_NAME}: Complex search query returned ${rows ? rows.length : 0} results`); + resolve(rows || []); + } + }); + }); + + // Format the results + const formattedResults = results.map(r => ({ + id: r.rule_id, + title: r.title || r.rule_id + })); + + logger.debug(`${FILE_NAME}: Returning ${formattedResults.length} results for complex search`); + + return { + results: formattedResults, + totalCount + }; + } catch (error) { + logger.error(`${FILE_NAME}: Error in complex search operation: ${error.message}`); + logger.debug(`${FILE_NAME}: Complex search error stack: ${error.stack}`); + return { results: [], totalCount: 0 }; + } finally { + // IMPORTANT: Only close the db connection if we're not using FTS + // When using FTS, let searchRulesComplexFTS manage the connection + if (db && !usingFts) { + try { + await new Promise((resolve) => db.close(() => resolve())); + logger.debug(`${FILE_NAME}: Database connection closed after complex search operation`); + } catch (closeError) { + logger.error(`${FILE_NAME}: Error closing database after complex search: ${closeError.message}`); + } + } + } +} + +/** + * Search for Sigma rules using complex query conditions with FTS5 + * Uses the FTS5 virtual table for faster text searching + * + * @param {Object} parsedQuery - The parsed query object + * @param {number} limit - Maximum number of results to return + * @param {number} offset - Number of results to skip (for pagination) + * @param {Object} providedDb - Database connection (optional, will create one if not provided) + * @returns {Promise} Object with results array and total count + */ +async function searchRulesComplexFTS(parsedQuery, limit = 10, offset = 0, providedDb = null) { + if (!parsedQuery || !parsedQuery.valid) { + logger.warn(`${FILE_NAME}: Invalid query object provided for FTS complex search`); + return { results: [], totalCount: 0 }; + } + + logger.info(`${FILE_NAME}: Performing complex FTS search with ${parsedQuery.conditions.length} conditions`); + + let db; + let shouldCloseDb = false; + + try { + // Use provided db connection or create a new one + if (providedDb) { + db = providedDb; + } else { + db = await getDbConnection(); + shouldCloseDb = true; + logger.debug(`${FILE_NAME}: Created new database connection for complex FTS search`); + } + + // Build FTS query from conditions + const { ftsQuery, whereClause, params } = buildComplexFtsQuery(parsedQuery); + + logger.debug(`${FILE_NAME}: FTS query: "${ftsQuery}", additional where: ${whereClause ? whereClause : 'none'}`); + logger.debug(`${FILE_NAME}: Query parameters: ${JSON.stringify(params)}`); + + // Build count query + let countQuery; + let countParams; + + if (whereClause) { + countQuery = ` + SELECT COUNT(*) as count + FROM rule_search + WHERE rule_search MATCH ? + AND ${whereClause} + `; + countParams = [ftsQuery, ...params]; + } else { + countQuery = ` + SELECT COUNT(*) as count + FROM rule_search + WHERE rule_search MATCH ? + `; + countParams = [ftsQuery]; + } + + // Get total count + const countResult = await new Promise((resolve, reject) => { + db.get(countQuery, countParams, (err, row) => { + if (err) { + logger.error(`${FILE_NAME}: Complex FTS count query error: ${err.message}`); + reject(err); + } else { + resolve(row || { count: 0 }); + } + }); + }); + + const totalCount = countResult.count; + logger.debug(`${FILE_NAME}: Total matching rules for complex FTS query: ${totalCount}`); + + // Build results query with pagination + let searchQuery; + let searchParams; + + if (whereClause) { + searchQuery = ` + SELECT rule_id, title + FROM rule_search + WHERE rule_search MATCH ? + AND ${whereClause} + ORDER BY rank + LIMIT ? OFFSET ? + `; + searchParams = [ftsQuery, ...params, limit, offset]; + } else { + searchQuery = ` + SELECT rule_id, title + FROM rule_search + WHERE rule_search MATCH ? + ORDER BY rank + LIMIT ? OFFSET ? + `; + searchParams = [ftsQuery, limit, offset]; + } + + // Get paginated results + const results = await new Promise((resolve, reject) => { + db.all(searchQuery, searchParams, (err, rows) => { + if (err) { + logger.error(`${FILE_NAME}: Complex FTS search query error: ${err.message}`); + reject(err); + } else { + logger.debug(`${FILE_NAME}: Complex FTS search query returned ${rows ? rows.length : 0} results`); + resolve(rows || []); + } + }); + }); + + // Format the results + const formattedResults = results.map(r => ({ + id: r.rule_id, + title: r.title || r.rule_id + })); + + logger.debug(`${FILE_NAME}: Returning ${formattedResults.length} results for complex FTS search`); + + return { + results: formattedResults, + totalCount + }; + } catch (error) { + logger.error(`${FILE_NAME}: Error in complex FTS search operation: ${error.message}`); + logger.debug(`${FILE_NAME}: Complex FTS search error stack: ${error.stack}`); + return { results: [], totalCount: 0 }; + } finally { + // Only close the database if we created it AND we're not in the middle of a transaction + if (db && shouldCloseDb) { + try { + await db.close(); + logger.debug(`${FILE_NAME}: Database connection closed after complex FTS search`); + } catch (closeError) { + logger.error(`${FILE_NAME}: Error closing database after complex FTS search: ${closeError.message}`); + } + } + } +} + +module.exports = { + searchRulesComplex, + searchRulesComplexFTS +}; \ No newline at end of file diff --git a/src/sigma_db/queries/fts-search.js b/src/sigma_db/queries/fts-search.js new file mode 100644 index 0000000..fdaea2b --- /dev/null +++ b/src/sigma_db/queries/fts-search.js @@ -0,0 +1,133 @@ +/** + * fts-search.js + * Functions for Full Text Search (FTS) of Sigma rules + */ + +const { getDbConnection } = require('../sigma_db_connection'); +const logger = require('../../utils/logger'); +const { getFileName } = require('../../utils/file_utils'); +const FILE_NAME = getFileName(__filename); + +/** + * Check if FTS5 virtual table is available + * + * @param {Object} db - Database connection + * @returns {Promise} Whether FTS5 is available + */ +async function checkFtsAvailable(db) { + try { + const result = await new Promise((resolve, reject) => { + db.get("SELECT name FROM sqlite_master WHERE type='table' AND name='rule_search'", (err, row) => { + if (err) { + logger.error(`${FILE_NAME}: Error checking for FTS5 table: ${err.message}`); + reject(err); + } else { + resolve(row !== undefined); + } + }); + }); + + logger.debug(`${FILE_NAME}: FTS5 table availability check: ${result ? 'Available' : 'Not available'}`); + return result; + } catch (error) { + logger.error(`${FILE_NAME}: Error checking FTS availability: ${error.message}`); + return false; + } +} + +/** + * Search for Sigma rules using FTS5 + * Performs a full-text search and returns matching rules with pagination + * + * @param {string} keyword - The keyword to search for + * @param {number} limit - Maximum number of results to return (default: 10) + * @param {number} offset - Number of results to skip (for pagination, default: 0) + * @returns {Promise} Object with results array and total count + */ +async function searchRulesFTS(keyword, limit = 10, offset = 0) { + if (!keyword) { + logger.warn(`${FILE_NAME}: Empty search keyword provided for FTS search`); + return { results: [], totalCount: 0 }; + } + + // Prepare FTS query - add * for prefix matching if not already present + let ftsQuery = keyword.trim(); + if (!ftsQuery.endsWith('*')) { + ftsQuery = `${ftsQuery}*`; + } + + logger.info(`${FILE_NAME}: Performing FTS search with query: "${ftsQuery}" (limit: ${limit}, offset: ${offset})`); + + let db; + try { + db = await getDbConnection(); + logger.debug(`${FILE_NAME}: Database connection established for FTS search`); + + // First get the total count of matching rules + const countQuery = ` + SELECT COUNT(*) as count + FROM rule_search + WHERE rule_search MATCH ? + `; + + const countResult = await new Promise((resolve, reject) => { + db.get(countQuery, [ftsQuery], (err, row) => { + if (err) { + logger.error(`${FILE_NAME}: FTS count query error: ${err.message}`); + reject(err); + } else { + resolve(row || { count: 0 }); + } + }); + }); + + const totalCount = countResult.count; + logger.debug(`${FILE_NAME}: Total matching rules for FTS query "${ftsQuery}": ${totalCount}`); + + // Now get the actual results with pagination + const searchQuery = ` + SELECT rule_id, title + FROM rule_search + WHERE rule_search MATCH ? + ORDER BY rank + LIMIT ? OFFSET ? + `; + + const results = await new Promise((resolve, reject) => { + db.all(searchQuery, [ftsQuery, limit, offset], (err, rows) => { + if (err) { + logger.error(`${FILE_NAME}: FTS search query error: ${err.message}`); + reject(err); + } else { + logger.debug(`${FILE_NAME}: FTS search query returned ${rows ? rows.length : 0} results`); + resolve(rows || []); + } + }); + }); + + logger.debug(`${FILE_NAME}: FTS search results page for query "${ftsQuery}": ${results.length} matches (page ${Math.floor(offset / limit) + 1})`); + + return { + results: results.map(r => ({ id: r.rule_id, title: r.title || r.rule_id })), + totalCount + }; + } catch (error) { + logger.error(`${FILE_NAME}: Error in FTS search operation: ${error.message}`); + logger.debug(`${FILE_NAME}: FTS search error stack: ${error.stack}`); + return { results: [], totalCount: 0 }; + } finally { + if (db) { + try { + await db.close(); + logger.debug(`${FILE_NAME}: Database connection closed after FTS search operation`); + } catch (closeError) { + logger.error(`${FILE_NAME}: Error closing database connection after FTS search: ${closeError.message}`); + } + } + } +} + +module.exports = { + checkFtsAvailable, + searchRulesFTS +}; \ No newline at end of file diff --git a/src/sigma_db/queries/index.js b/src/sigma_db/queries/index.js new file mode 100644 index 0000000..b3e9894 --- /dev/null +++ b/src/sigma_db/queries/index.js @@ -0,0 +1,34 @@ +/** + * index.js + * Central module for accessing all Sigma database query functions + */ + +// Import functions from individual modules +const ruleRetrieval = require('./rule-retrieval'); +const simpleSearch = require('./simple-search'); +const ftsSearch = require('./fts-search'); +const complexSearch = require('./complex-search'); +const statsDebug = require('./stats-debug'); + +// Export all functions +module.exports = { + // Rule retrieval functions + getAllRuleIds: ruleRetrieval.getAllRuleIds, + findRuleById: ruleRetrieval.findRuleById, + + // Search functions + searchRules: simpleSearch.searchRules, + + // FTS search functions + searchRulesFTS: ftsSearch.searchRulesFTS, + checkFtsAvailable: ftsSearch.checkFtsAvailable, + + // Complex search functions + searchRulesComplex: complexSearch.searchRulesComplex, + searchRulesComplexFTS: complexSearch.searchRulesComplexFTS, + + // Stats and debug functions + debugRuleContent: statsDebug.debugRuleContent, + getRuleYamlContent: statsDebug.getRuleYamlContent, + getStatsFromDatabase: statsDebug.getStatsFromDatabase +}; \ No newline at end of file diff --git a/src/sigma_db/queries/query-builders.js b/src/sigma_db/queries/query-builders.js new file mode 100644 index 0000000..0224d38 --- /dev/null +++ b/src/sigma_db/queries/query-builders.js @@ -0,0 +1,258 @@ +/** + * query-builders.js + * Helper functions for building SQL queries + */ + +const logger = require('../../utils/logger'); +const { getFileName } = require('../../utils/file_utils'); +const FILE_NAME = getFileName(__filename); + +/** + * Build the SQL query for complex search based on parsed conditions + * + * @param {Object} parsedQuery - The parsed query object + * @param {number} limit - Results limit + * @param {number} offset - Results offset + * @returns {Object} Object with SQL query, count query, and parameters + */ +function buildComplexSqlQuery(parsedQuery, limit, offset) { + const { conditions, operator } = parsedQuery; + const params = []; + + // Start building the primary table selection + let sqlSelectPart = ` + SELECT DISTINCT r.id as rule_id, + (SELECT param_value FROM rule_parameters WHERE rule_id = r.id AND param_name = 'title' LIMIT 1) as title + FROM sigma_rules r + `; + + // Build WHERE clause based on conditions + let whereClauses = []; + let joinIdx = 0; + + for (const condition of conditions) { + let whereClause = ''; + + switch (condition.field) { + case 'title': + joinIdx++; + whereClause = `EXISTS ( + SELECT 1 FROM rule_parameters p${joinIdx} + WHERE p${joinIdx}.rule_id = r.id + AND p${joinIdx}.param_name = 'title' + AND INSTR(LOWER(p${joinIdx}.param_value), LOWER(?)) > 0 + )`; + params.push(condition.value); + break; + + case 'description': + joinIdx++; + whereClause = `EXISTS ( + SELECT 1 FROM rule_parameters p${joinIdx} + WHERE p${joinIdx}.rule_id = r.id + AND p${joinIdx}.param_name = 'description' + AND INSTR(LOWER(p${joinIdx}.param_value), LOWER(?)) > 0 + )`; + params.push(condition.value); + break; + + case 'logsource': + joinIdx++; + whereClause = `EXISTS ( + SELECT 1 FROM rule_parameters p${joinIdx} + WHERE p${joinIdx}.rule_id = r.id + AND p${joinIdx}.param_name = 'logsource' + AND INSTR(LOWER(p${joinIdx}.param_value), LOWER('"${condition.subfield}":"${condition.value}"')) > 0 + )`; + break; + + case 'tags': + joinIdx++; + whereClause = `EXISTS ( + SELECT 1 FROM rule_parameters p${joinIdx} + WHERE p${joinIdx}.rule_id = r.id + AND p${joinIdx}.param_name = 'tags' + AND INSTR(LOWER(p${joinIdx}.param_value), LOWER(?)) > 0 + )`; + params.push(condition.value); + break; + + case 'date': + joinIdx++; + const dateOperator = condition.operator === 'after' ? '>' : + condition.operator === 'before' ? '<' : '='; + whereClause = `r.date ${dateOperator} date(?)`; + params.push(condition.value); + break; + + case 'author': + joinIdx++; + whereClause = `EXISTS ( + SELECT 1 FROM rule_parameters p${joinIdx} + WHERE p${joinIdx}.rule_id = r.id + AND p${joinIdx}.param_name = 'author' + AND INSTR(LOWER(p${joinIdx}.param_value), LOWER(?)) > 0 + )`; + params.push(condition.value); + break; + + case 'level': + joinIdx++; + whereClause = `EXISTS ( + SELECT 1 FROM rule_parameters p${joinIdx} + WHERE p${joinIdx}.rule_id = r.id + AND p${joinIdx}.param_name = 'level' + AND LOWER(p${joinIdx}.param_value) = LOWER(?) + )`; + params.push(condition.value); + break; + + case 'id': + whereClause = `LOWER(r.id) = LOWER(?)`; + params.push(condition.value); + break; + + case 'keyword': + default: + // Default to searching in title + joinIdx++; + whereClause = `EXISTS ( + SELECT 1 FROM rule_parameters p${joinIdx} + WHERE p${joinIdx}.rule_id = r.id + AND p${joinIdx}.param_name = 'title' + AND INSTR(LOWER(p${joinIdx}.param_value), LOWER(?)) > 0 + )`; + params.push(condition.value); + break; + } + + if (whereClause) { + whereClauses.push(whereClause); + } + } + + // Combine the WHERE clauses with the appropriate operator + let whereStatement = ''; + if (whereClauses.length > 0) { + const combiner = operator === 'AND' ? ' AND ' : ' OR '; + whereStatement = `WHERE ${whereClauses.join(combiner)}`; + } + + // Complete queries + const sqlQuery = ` + ${sqlSelectPart} + ${whereStatement} + ORDER BY rule_id + LIMIT ? OFFSET ? + `; + + const sqlCountQuery = ` + SELECT COUNT(DISTINCT r.id) as count + FROM sigma_rules r + ${whereStatement} + `; + + // Add pagination parameters + params.push(limit); + params.push(offset); + + return { sqlQuery, sqlCountQuery, params }; +} + +/** + * Build FTS query and WHERE clause from parsed query conditions + * + * @param {Object} parsedQuery - The parsed query object + * @returns {Object} Object with FTS query, additional WHERE clause, and parameters + */ +function buildComplexFtsQuery(parsedQuery) { + const { conditions, operator } = parsedQuery; + + // Separate text search conditions from other conditions + const textConditions = []; + const nonTextConditions = []; + + for (const condition of conditions) { + switch (condition.field) { + case 'title': + case 'description': + case 'author': + case 'tags': + case 'keyword': + // These can be handled by FTS directly + textConditions.push(condition); + break; + default: + // These need additional WHERE clauses + nonTextConditions.push(condition); + break; + } + } + + // Build FTS MATCH query + let ftsQueryParts = []; + + for (const condition of textConditions) { + let fieldPrefix = ''; + + // Add field-specific prefix if available + if (condition.field !== 'keyword') { + fieldPrefix = `${condition.field}:`; + } + + // Add wildcard for partial matching if not already present + let value = condition.value.trim(); + if (!value.endsWith('*')) { + value = `${value}*`; + } + + ftsQueryParts.push(`${fieldPrefix}${value}`); + } + + // If no text conditions, use a match-all query + const ftsQuery = ftsQueryParts.length > 0 + ? ftsQueryParts.join(operator === 'AND' ? ' AND ' : ' OR ') + : '*'; + + // Build additional WHERE clauses for non-text conditions + let whereClauseParts = []; + const params = []; + + for (const condition of nonTextConditions) { + switch (condition.field) { + case 'date': + const dateOperator = condition.operator === 'after' ? '>' : + condition.operator === 'before' ? '<' : '='; + whereClauseParts.push(`date ${dateOperator} date(?)`); + params.push(condition.value); + break; + + case 'level': + whereClauseParts.push(`level = ?`); + params.push(condition.value); + break; + + case 'logsource': + whereClauseParts.push(`logsource LIKE ?`); + params.push(`%${condition.subfield}%${condition.value}%`); + break; + + case 'id': + whereClauseParts.push(`rule_id = ?`); + params.push(condition.value); + break; + } + } + + // Combine WHERE clauses + const whereClause = whereClauseParts.length > 0 + ? whereClauseParts.join(operator === 'AND' ? ' AND ' : ' OR ') + : ''; + + return { ftsQuery, whereClause, params }; +} + +module.exports = { + buildComplexSqlQuery, + buildComplexFtsQuery +}; \ No newline at end of file diff --git a/src/sigma_db/queries/rule-retrieval.js b/src/sigma_db/queries/rule-retrieval.js new file mode 100644 index 0000000..10a0a43 --- /dev/null +++ b/src/sigma_db/queries/rule-retrieval.js @@ -0,0 +1,164 @@ +/** + * rule-retrieval.js + * Functions for retrieving Sigma rules and rule IDs + */ + +const { getDbConnection } = require('../sigma_db_connection'); +const logger = require('../../utils/logger'); +const { getFileName } = require('../../utils/file_utils'); +const FILE_NAME = getFileName(__filename); + +/** + * Get a list of all rule IDs in the database + * Useful for bulk operations and database integrity checks + * + * @returns {Promise} Array of rule IDs or empty array on error + */ +async function getAllRuleIds() { + let db; + try { + logger.info(`${FILE_NAME}: Retrieving all rule IDs from database`); + + db = await getDbConnection(); + logger.debug(`${FILE_NAME}: Connected to database for retrieving all rule IDs`); + + const result = await new Promise((resolve, reject) => { + db.all('SELECT id FROM sigma_rules ORDER BY id', [], (err, rows) => { + if (err) { + logger.error(`${FILE_NAME}: Error fetching all rule IDs: ${err.message}`); + reject(err); + } else { + resolve(rows || []); + } + }); + }); + + logger.debug(`${FILE_NAME}: Retrieved ${result.length} rule IDs from database`); + return result.map(row => row.id); + } catch (error) { + logger.error(`${FILE_NAME}: Error retrieving all rule IDs: ${error.message}`); + logger.debug(`${FILE_NAME}: Error stack: ${error.stack}`); + return []; + } finally { + if (db) { + try { + await db.close(); + logger.debug(`${FILE_NAME}: Database connection closed after retrieving all rule IDs`); + } catch (closeError) { + logger.warn(`${FILE_NAME}: Error closing database: ${closeError.message}`); + } + } + } +} + +/** + * Find a Sigma rule by its ID + * Retrieves rule data and associated parameters from the database + * + * @param {string} ruleId - The ID of the rule to find + * @returns {Promise} The rule object or null if not found + */ +async function findRuleById(ruleId) { + if (!ruleId) { + logger.warn(`${FILE_NAME}: Cannot find rule: Missing rule ID`); + return null; + } + + let db; + try { + db = await getDbConnection(); + logger.debug(`${FILE_NAME}: Connected to database for rule lookup: ${ruleId}`); + + // Get the base rule using promisified method + const rule = await db.getAsync('SELECT * FROM sigma_rules WHERE id = ?', [ruleId]); + if (!rule) { + logger.warn(`${FILE_NAME}: Rule with ID ${ruleId} not found in database`); + return null; + } + + logger.debug(`${FILE_NAME}: Found base rule with ID ${ruleId}, content length: ${rule.content ? rule.content.length : 0}`); + + // Get parameters using promisified method + const paramsAsync = await db.allAsync('SELECT param_name, param_value, param_type FROM rule_parameters WHERE rule_id = ?', [ruleId]); + logger.debug(`${FILE_NAME}: Params query returned ${paramsAsync ? paramsAsync.length : 0} results via allAsync`); + + // Check if content is missing + if (!rule.content) { + logger.warn(`${FILE_NAME}: Rule with ID ${ruleId} exists but has no content`); + rule.content_missing = true; + } + + // Get all parameters for this rule with case-insensitive matching + try { + const params = await new Promise((resolve, reject) => { + db.all( + 'SELECT param_name, param_value, param_type FROM rule_parameters WHERE LOWER(rule_id) = LOWER(?)', + [ruleId], + (err, rows) => { + if (err) reject(err); + else resolve(rows); + } + ); + }); + + logger.debug(`${FILE_NAME}: Retrieved ${params ? params.length : 0} parameters for rule ${ruleId}`); + + // Validate params is an array + if (params && Array.isArray(params)) { + // Attach parameters to the rule object + rule.parameters = {}; + + for (const param of params) { + if (param && param.param_name) { + // Convert value based on type + let value = param.param_value; + + if (param.param_type === 'object' || param.param_type === 'array') { + try { + value = JSON.parse(param.param_value); + } catch (parseError) { + logger.warn(`${FILE_NAME}: Failed to parse JSON for parameter ${param.param_name}: ${parseError.message}`); + } + } else if (param.param_type === 'boolean') { + value = param.param_value === 'true'; + } else if (param.param_type === 'number') { + value = Number(param.param_value); + } + + rule.parameters[param.param_name] = value; + } + } + + logger.debug(`${FILE_NAME}: Successfully processed ${Object.keys(rule.parameters).length} parameters for rule ${ruleId}`); + } else { + logger.warn(`${FILE_NAME}: Parameters for rule ${ruleId} not available or not iterable`); + rule.parameters = {}; + } + } catch (paramError) { + logger.error(`${FILE_NAME}: Error fetching parameters for rule ${ruleId}: ${paramError.message}`); + logger.debug(`${FILE_NAME}: Parameter error stack: ${paramError.stack}`); + rule.parameters = {}; + } + + return rule; + } catch (error) { + logger.error(`${FILE_NAME}: Error finding rule ${ruleId}: ${error.message}`); + logger.debug(`${FILE_NAME}: Error stack: ${error.stack}`); + return null; + } finally { + // Close the database connection if it was opened + if (db && typeof db.close === 'function') { + try { + await db.close(); + logger.debug(`${FILE_NAME}: Database connection closed after rule lookup`); + } catch (closeError) { + logger.warn(`${FILE_NAME}: Error closing database connection: ${closeError.message}`); + } + } + } +} + +module.exports = { + getAllRuleIds, + findRuleById +}; \ No newline at end of file diff --git a/src/sigma_db/queries/simple-search.js b/src/sigma_db/queries/simple-search.js new file mode 100644 index 0000000..453a16d --- /dev/null +++ b/src/sigma_db/queries/simple-search.js @@ -0,0 +1,118 @@ +/** + * simple-search.js + * Functions for basic search of Sigma rules + */ + +const { getDbConnection } = require('../sigma_db_connection'); +const logger = require('../../utils/logger'); +const { getFileName } = require('../../utils/file_utils'); +const FILE_NAME = getFileName(__filename); + +// Import FTS functions - need to use relative path for proper circular dependency handling +const { checkFtsAvailable, searchRulesFTS } = require('./fts-search'); + +/** + * Search for Sigma rules by keyword in rule titles + * Performs a case-insensitive search and returns matching rules with pagination + * + * @param {string} keyword - The keyword to search for + * @param {number} limit - Maximum number of results to return (default: 10) + * @param {number} offset - Number of results to skip (for pagination, default: 0) + * @returns {Promise} Object with results array and total count + */ +async function searchRules(keyword, limit = 10, offset = 0) { + if (!keyword) { + logger.warn(`${FILE_NAME}: Empty search keyword provided`); + return { results: [], totalCount: 0 }; + } + + // Sanitize keyword to prevent SQL injection + const sanitizedKeyword = keyword.replace(/'/g, "''"); + logger.info(`${FILE_NAME}: Searching for rules with keyword in title: ${sanitizedKeyword} (limit: ${limit}, offset: ${offset})`); + + let db; + try { + // Make sure we properly await the DB connection + db = await getDbConnection(); + logger.debug(`${FILE_NAME}: Database connection established for search`); + + // Use FTS5 for faster searching if available + const ftsAvailable = await checkFtsAvailable(db); + + if (ftsAvailable) { + logger.debug(`${FILE_NAME}: Using FTS5 for keyword search`); + return searchRulesFTS(keyword, limit, offset); + } + + // If FTS5 is not available, use the legacy search method + logger.debug(`${FILE_NAME}: FTS5 not available, using legacy search method`); + + // First get the total count of matching rules (for pagination info) + const countQuery = ` + SELECT COUNT(*) as count + FROM rule_parameters + WHERE param_name = 'title' + AND INSTR(LOWER(param_value), LOWER(?)) > 0 + `; + + const countResult = await new Promise((resolve, reject) => { + db.get(countQuery, [sanitizedKeyword], (err, row) => { + if (err) { + logger.error(`${FILE_NAME}: Count query error: ${err.message}`); + reject(err); + } else { + resolve(row || { count: 0 }); + } + }); + }); + + const totalCount = countResult.count; + logger.debug(`${FILE_NAME}: Total matching rules for "${sanitizedKeyword}": ${totalCount}`); + + // Use parameterized query instead of string interpolation for better security + const instrQuery = ` + SELECT rule_id, param_value AS title + FROM rule_parameters + WHERE param_name = 'title' + AND INSTR(LOWER(param_value), LOWER(?)) > 0 + LIMIT ? OFFSET ? + `; + + const results = await new Promise((resolve, reject) => { + db.all(instrQuery, [sanitizedKeyword, limit, offset], (err, rows) => { + if (err) { + logger.error(`${FILE_NAME}: Search query error: ${err.message}`); + reject(err); + } else { + logger.debug(`${FILE_NAME}: Search query returned ${rows ? rows.length : 0} results`); + resolve(rows || []); + } + }); + }); + + logger.debug(`${FILE_NAME}: Search results page for keyword "${sanitizedKeyword}": ${results.length} matches (page ${Math.floor(offset / limit) + 1})`); + + return { + results: results.map(r => ({ id: r.rule_id, title: r.title })), + totalCount + }; + } catch (error) { + logger.error(`${FILE_NAME}: Error in search operation: ${error.message}`); + logger.debug(`${FILE_NAME}: Search error stack: ${error.stack}`); + return { results: [], totalCount: 0 }; + } finally { + // Make sure we properly close the connection + if (db) { + try { + await new Promise((resolve) => db.close(() => resolve())); + logger.debug(`${FILE_NAME}: Database connection closed after search operation`); + } catch (closeError) { + logger.error(`${FILE_NAME}: Error closing database connection after search: ${closeError.message}`); + } + } + } +} + +module.exports = { + searchRules +}; \ No newline at end of file diff --git a/src/sigma_db/queries/stats-debug.js b/src/sigma_db/queries/stats-debug.js new file mode 100644 index 0000000..e8fda9c --- /dev/null +++ b/src/sigma_db/queries/stats-debug.js @@ -0,0 +1,328 @@ +/** + * stats-debug.js + * Functions for database statistics and debugging + */ + +const { getDbConnection } = require('../sigma_db_connection'); +const logger = require('../../utils/logger'); +const { DB_PATH } = require('../../config/appConfig'); +const path = require('path'); +const { getFileName } = require('../../utils/file_utils'); +const FILE_NAME = getFileName(__filename); + +/** + * Debug function to retrieve detailed information about a rule's content + * Useful for diagnosing issues with rule retrieval and content parsing + * + * @param {string} ruleId - The ID of the rule to debug + * @returns {Promise} Object containing debug information or null on error + */ +async function debugRuleContent(ruleId) { + if (!ruleId) { + logger.warn(`${FILE_NAME}: Cannot debug rule: Missing rule ID`); + return null; + } + + let db; + try { + db = await getDbConnection(); + + const absolutePath = path.resolve(DB_PATH); + logger.debug(`${FILE_NAME}: Debug function connecting to DB at path: ${absolutePath}`); + + // Get raw rule record + const rule = await db.get('SELECT id, file_path, length(content) as content_length, typeof(content) as content_type FROM sigma_rules WHERE id = ?', [ruleId]); + + if (!rule) { + logger.warn(`${FILE_NAME}: Rule with ID ${ruleId} not found during debug operation`); + return { error: 'Rule not found', ruleId }; + } + + // Return just the rule information without the undefined variables + return { + rule, + ruleId + }; + } catch (error) { + logger.error(`${FILE_NAME}: Debug error for rule ${ruleId}: ${error.message}`); + logger.debug(`${FILE_NAME}: Debug error stack: ${error.stack}`); + return { + error: error.message, + stack: error.stack, + ruleId + }; + } finally { + if (db) { + try { + await db.close(); + logger.debug(`${FILE_NAME}: Database connection closed after debug operation`); + } catch (closeError) { + logger.warn(`${FILE_NAME}: Error closing database after debug: ${closeError.message}`); + } + } + } +} + +/** + * Get the raw YAML content of a Sigma rule + * Retrieves the content field from the database which should contain YAML + * + * @param {string} ruleId - The ID of the rule + * @returns {Promise} Object with success flag and content or error message + */ +async function getRuleYamlContent(ruleId) { + if (!ruleId) { + logger.warn(`${FILE_NAME}: Cannot get YAML content: Missing rule ID`); + return { success: false, message: 'Missing rule ID' }; + } + + let db; + try { + logger.info(`${FILE_NAME}: Fetching YAML content for rule: ${ruleId}`); + logger.debug(`${FILE_NAME}: Rule ID type: ${typeof ruleId}, length: ${ruleId.length}`); + + db = await getDbConnection(); + logger.debug(`${FILE_NAME}: Connected to database for YAML retrieval`); + + // Debug query before execution + const debugResult = await db.get('SELECT id, typeof(content) as content_type, length(content) as content_length FROM sigma_rules WHERE id = ?', [ruleId]); + logger.debug(`${FILE_NAME}: Debug query result: ${JSON.stringify(debugResult || 'not found')}`); + + if (!debugResult) { + logger.warn(`${FILE_NAME}: Rule with ID ${ruleId} not found in debug query`); + return { success: false, message: 'Rule not found' }; + } + + // Get actual content + const rule = await new Promise((resolve, reject) => { + db.get('SELECT content FROM sigma_rules WHERE id = ?', [ruleId], (err, row) => { + if (err) { + logger.error(`${FILE_NAME}: Content query error: ${err.message}`); + reject(err); + } else { + resolve(row || null); + } + }); + }); + logger.debug(`${FILE_NAME}: Content query result for ${ruleId}: ${rule ? 'Found' : 'Not found'}`); + + if (!rule) { + logger.warn(`${FILE_NAME}: Rule with ID ${ruleId} not found in content query`); + return { success: false, message: 'Rule not found' }; + } + + if (!rule.content) { + logger.warn(`${FILE_NAME}: Rule with ID ${ruleId} found but has no content`); + return { success: false, message: 'Rule found but content is empty' }; + } + + logger.debug(`${FILE_NAME}: Content retrieved successfully for ${ruleId}, type: ${typeof rule.content}, length: ${rule.content.length}`); + + return { success: true, content: rule.content }; + } catch (error) { + logger.error(`${FILE_NAME}: Error retrieving YAML content for ${ruleId}: ${error.message}`); + logger.debug(`${FILE_NAME}: YAML retrieval error stack: ${error.stack}`); + return { success: false, message: `Error retrieving YAML: ${error.message}` }; + } finally { + if (db) { + try { + await db.close(); + logger.debug(`${FILE_NAME}: Database connection closed after YAML retrieval`); + } catch (closeError) { + logger.warn(`${FILE_NAME}: Error closing database after YAML retrieval: ${closeError.message}`); + } + } + } +} + +/** + * Get statistics about Sigma rules in the database + * Collects counts, categories, and other aggregate information + * + * @returns {Promise} Object with various statistics about the rules + */ +async function getStatsFromDatabase() { + let db; + try { + db = await getDbConnection(); + logger.debug(`${FILE_NAME}: Connected to database for statistics`); + + // Get total rule count + const totalRulesResult = await new Promise((resolve, reject) => { + db.get('SELECT COUNT(*) as count FROM sigma_rules', (err, row) => { + if (err) reject(err); + else resolve(row); + }); + }); + const totalRules = totalRulesResult.count; + + // Get last update time + const lastUpdateResult = await new Promise((resolve, reject) => { + db.get('SELECT MAX(date) as last_update FROM sigma_rules', (err, row) => { + if (err) reject(err); + else resolve(row); + }); + }); + const lastUpdate = lastUpdateResult.last_update; + + // Get rules by log source count (Windows, Linux, macOS) + const windowsRulesResult = await new Promise((resolve, reject) => { + db.get(` + SELECT COUNT(DISTINCT rule_id) as count + FROM rule_parameters + WHERE param_name = 'logsource' AND + param_value LIKE '%"product":"windows"%'`, + (err, row) => { + if (err) reject(err); + else resolve(row); + }); + }); + const windowsRules = windowsRulesResult.count || 0; + + const linuxRulesResult = await new Promise((resolve, reject) => { + db.get(` + SELECT COUNT(DISTINCT rule_id) as count + FROM rule_parameters + WHERE param_name = 'logsource' AND + param_value LIKE '%"product":"linux"%'`, + (err, row) => { + if (err) reject(err); + else resolve(row); + }); + }); + const linuxRules = linuxRulesResult.count || 0; + + const macosRulesResult = await new Promise((resolve, reject) => { + db.get(` + SELECT COUNT(DISTINCT rule_id) as count + FROM rule_parameters + WHERE param_name = 'logsource' AND + param_value LIKE '%"product":"macos"%'`, + (err, row) => { + if (err) reject(err); + else resolve(row); + }); + }); + const macosRules = macosRulesResult.count || 0; + + // Get rules by severity level + const severityStats = await new Promise((resolve, reject) => { + db.all(` + SELECT param_value AS level, COUNT(DISTINCT rule_id) as count + FROM rule_parameters + WHERE param_name = 'level' + GROUP BY param_value + ORDER BY + CASE + WHEN param_value = 'critical' THEN 1 + WHEN param_value = 'high' THEN 2 + WHEN param_value = 'medium' THEN 3 + WHEN param_value = 'low' THEN 4 + WHEN param_value = 'informational' THEN 5 + ELSE 6 + END`, + (err, rows) => { + if (err) reject(err); + else resolve(rows); + }); + }); + + // Get top 5 rule authors + const topAuthors = await new Promise((resolve, reject) => { + db.all(` + SELECT param_value AS author, COUNT(*) as count + FROM rule_parameters + WHERE param_name = 'author' + GROUP BY param_value + ORDER BY count DESC + LIMIT 5`, + (err, rows) => { + if (err) reject(err); + else resolve(rows); + }); + }); + + // Get empty content count (rules with missing YAML) + const emptyContentResult = await new Promise((resolve, reject) => { + db.get(` + SELECT COUNT(*) as count + FROM sigma_rules + WHERE content IS NULL OR content = ''`, + (err, row) => { + if (err) reject(err); + else resolve(row); + }); + }); + const emptyContentCount = emptyContentResult.count; + + // Get MITRE ATT&CK tactics statistics + const mitreStats = await new Promise((resolve, reject) => { + db.all(` + SELECT param_value AS tag, COUNT(DISTINCT rule_id) as count + FROM rule_parameters + WHERE param_name = 'tags' AND param_value LIKE 'attack.%' + GROUP BY param_value + ORDER BY count DESC + LIMIT 10`, + (err, rows) => { + if (err) reject(err); + else resolve(rows); + }); + }); + + // Format MITRE tactics for display + const formattedMitreTactics = mitreStats.map(item => { + const tactic = item.tag.substring(7); // Remove 'attack.' prefix + return { + tactic: tactic, + count: item.count + }; + }); + + // Compile all statistics + const stats = { + totalRules, + lastUpdate, + operatingSystems: { + windows: windowsRules, + linux: linuxRules, + macos: macosRules, + other: totalRules - (windowsRules + linuxRules + macosRules) + }, + severityLevels: severityStats.map(s => ({ level: s.level, count: s.count })), + topAuthors: topAuthors.map(a => ({ name: a.author, count: a.count })), + databaseHealth: { + emptyContentCount, + contentPercentage: totalRules > 0 ? Math.round(((totalRules - emptyContentCount) / totalRules) * 100) : 0 + }, + mitreTactics: formattedMitreTactics + }; + + return { + success: true, + stats + }; + } catch (error) { + logger.error(`${FILE_NAME}: Error retrieving statistics: ${error.message}`); + logger.debug(`${FILE_NAME}: Error stack: ${error.stack}`); + return { + success: false, + message: `Error retrieving statistics: ${error.message}` + }; + } finally { + if (db) { + try { + await new Promise((resolve) => db.close(() => resolve())); + logger.debug(`${FILE_NAME}: Database connection closed after statistics retrieval`); + } catch (closeError) { + logger.warn(`${FILE_NAME}: Error closing database: ${closeError.message}`); + } + } + } +} + +module.exports = { + debugRuleContent, + getRuleYamlContent, + getStatsFromDatabase +}; \ No newline at end of file diff --git a/src/sigma_db/sigma_db_queries.js b/src/sigma_db/sigma_db_queries.js deleted file mode 100644 index 8fa53f9..0000000 --- a/src/sigma_db/sigma_db_queries.js +++ /dev/null @@ -1,1207 +0,0 @@ -/** - * - * sigma_db_queries.js - * this script contains functions to interact with the Sigma database - * - * IMPORTANT: - * SQLite queries need explicit Promise handling when using db.all() - * - * We had an issue in that the Promise returned by db.all() wasn't being - * properly resolved in the async context. By wrapping the db.all() call in - * a new Promise and explicitly handling the callback, we ensure the query - * completes before continuing. This is important with SQLite where the - * connection state management can sometimes be tricky with async/await. - * - */ -const { getDbConnection } = require('./sigma_db_connection'); -const logger = require('../utils/logger'); -const { DB_PATH } = require('../config/appConfig'); -const path = require('path'); - -const { getFileName } = require('../utils/file_utils'); -const FILE_NAME = getFileName(__filename); - - -/** - * Get a list of all rule IDs in the database - * Useful for bulk operations and database integrity checks - * - * @returns {Promise} Array of rule IDs or empty array on error - */ -async function getAllRuleIds() { - let db; - try { - logger.info(`${FILE_NAME}: Retrieving all rule IDs from database`); - - db = await getDbConnection(); - logger.debug(`${FILE_NAME}: Connected to database for retrieving all rule IDs`); - - const result = await new Promise((resolve, reject) => { - db.all('SELECT id FROM sigma_rules ORDER BY id', [], (err, rows) => { - if (err) { - logger.error(`${FILE_NAME}: Error fetching all rule IDs: ${err.message}`); - reject(err); - } else { - resolve(rows || []); - } - }); - }); - - logger.debug(`${FILE_NAME}: Retrieved ${result.length} rule IDs from database`); - return result.map(row => row.id); - } catch (error) { - logger.error(`${FILE_NAME}: Error retrieving all rule IDs: ${error.message}`); - logger.debug(`${FILE_NAME}: Error stack: ${error.stack}`); - return []; - } finally { - if (db) { - try { - await db.close(); - logger.debug(`${FILE_NAME}: Database connection closed after retrieving all rule IDs`); - } catch (closeError) { - logger.warn(`${FILE_NAME}: Error closing database: ${closeError.message}`); - } - } - } -} - - -/** - * Find a Sigma rule by its ID - * Retrieves rule data and associated parameters from the database - * - * @param {string} ruleId - The ID of the rule to find - * @returns {Promise} The rule object or null if not found - */ -async function findRuleById(ruleId) { - if (!ruleId) { - logger.warn(`${FILE_NAME}: Cannot find rule: Missing rule ID`); - return null; - } - - let db; - try { - db = await getDbConnection(); - logger.debug(`${FILE_NAME}: Connected to database for rule lookup: ${ruleId}`); - - // Get the base rule using promisified method - const rule = await db.getAsync('SELECT * FROM sigma_rules WHERE id = ?', [ruleId]); - if (!rule) { - logger.warn(`${FILE_NAME}: Rule with ID ${ruleId} not found in database`); - return null; - } - - logger.debug(`${FILE_NAME}: Found base rule with ID ${ruleId}, content length: ${rule.content ? rule.content.length : 0}`); - - // Get parameters using promisified method - const paramsAsync = await db.allAsync('SELECT param_name, param_value, param_type FROM rule_parameters WHERE rule_id = ?', [ruleId]); - logger.debug(`${FILE_NAME}: Params query returned ${paramsAsync ? paramsAsync.length : 0} results via allAsync`); - - // Check if content is missing - if (!rule.content) { - logger.warn(`${FILE_NAME}: Rule with ID ${ruleId} exists but has no content`); - rule.content_missing = true; - } - - // Get all parameters for this rule with case-insensitive matching - try { - const params = await new Promise((resolve, reject) => { - db.all( - 'SELECT param_name, param_value, param_type FROM rule_parameters WHERE LOWER(rule_id) = LOWER(?)', - [ruleId], - (err, rows) => { - if (err) reject(err); - else resolve(rows); - } - ); - }); - - logger.debug(`${FILE_NAME}: Retrieved ${params ? params.length : 0} parameters for rule ${ruleId}`); - - // Validate params is an array - if (params && Array.isArray(params)) { - // Attach parameters to the rule object - rule.parameters = {}; - - for (const param of params) { - if (param && param.param_name) { - // Convert value based on type - let value = param.param_value; - - if (param.param_type === 'object' || param.param_type === 'array') { - try { - value = JSON.parse(param.param_value); - } catch (parseError) { - logger.warn(`${FILE_NAME}: Failed to parse JSON for parameter ${param.param_name}: ${parseError.message}`); - } - } else if (param.param_type === 'boolean') { - value = param.param_value === 'true'; - } else if (param.param_type === 'number') { - value = Number(param.param_value); - } - - rule.parameters[param.param_name] = value; - } - } - - logger.debug(`${FILE_NAME}: Successfully processed ${Object.keys(rule.parameters).length} parameters for rule ${ruleId}`); - } else { - logger.warn(`${FILE_NAME}: Parameters for rule ${ruleId} not available or not iterable`); - rule.parameters = {}; - } - } catch (paramError) { - logger.error(`${FILE_NAME}: Error fetching parameters for rule ${ruleId}: ${paramError.message}`); - logger.debug(`${FILE_NAME}: Parameter error stack: ${paramError.stack}`); - rule.parameters = {}; - } - - return rule; - } catch (error) { - logger.error(`${FILE_NAME}: Error finding rule ${ruleId}: ${error.message}`); - logger.debug(`${FILE_NAME}: Error stack: ${error.stack}`); - return null; - } finally { - // Close the database connection if it was opened - if (db && typeof db.close === 'function') { - try { - await db.close(); - logger.debug(`${FILE_NAME}: Database connection closed after rule lookup`); - } catch (closeError) { - logger.warn(`${FILE_NAME}: Error closing database connection: ${closeError.message}`); - } - } - } -} - -/** - * Search for Sigma rules by keyword in rule titles - * Performs a case-insensitive search and returns matching rules with pagination - * - * @param {string} keyword - The keyword to search for - * @param {number} limit - Maximum number of results to return (default: 10) - * @param {number} offset - Number of results to skip (for pagination, default: 0) - * @returns {Promise} Object with results array and total count - */ -async function searchRules(keyword, limit = 10, offset = 0) { - if (!keyword) { - logger.warn(`${FILE_NAME}: Empty search keyword provided`); - return { results: [], totalCount: 0 }; - } - - // Sanitize keyword to prevent SQL injection - const sanitizedKeyword = keyword.replace(/'/g, "''"); - logger.info(`${FILE_NAME}: Searching for rules with keyword in title: ${sanitizedKeyword} (limit: ${limit}, offset: ${offset})`); - - let db; - try { - // Make sure we properly await the DB connection - db = await getDbConnection(); - logger.debug(`${FILE_NAME}: Database connection established for search`); - - // Use FTS5 for faster searching if available - const ftsAvailable = await checkFtsAvailable(db); - - if (ftsAvailable) { - logger.debug(`${FILE_NAME}: Using FTS5 for keyword search`); - return searchRulesFTS(keyword, limit, offset); - } - - // If FTS5 is not available, use the legacy search method - logger.debug(`${FILE_NAME}: FTS5 not available, using legacy search method`); - - // First get the total count of matching rules (for pagination info) - const countQuery = ` - SELECT COUNT(*) as count - FROM rule_parameters - WHERE param_name = 'title' - AND INSTR(LOWER(param_value), LOWER(?)) > 0 - `; - - const countResult = await new Promise((resolve, reject) => { - db.get(countQuery, [sanitizedKeyword], (err, row) => { - if (err) { - logger.error(`${FILE_NAME}: Count query error: ${err.message}`); - reject(err); - } else { - resolve(row || { count: 0 }); - } - }); - }); - - const totalCount = countResult.count; - logger.debug(`${FILE_NAME}: Total matching rules for "${sanitizedKeyword}": ${totalCount}`); - - // Use parameterized query instead of string interpolation for better security - const instrQuery = ` - SELECT rule_id, param_value AS title - FROM rule_parameters - WHERE param_name = 'title' - AND INSTR(LOWER(param_value), LOWER(?)) > 0 - LIMIT ? OFFSET ? - `; - - const results = await new Promise((resolve, reject) => { - db.all(instrQuery, [sanitizedKeyword, limit, offset], (err, rows) => { - if (err) { - logger.error(`${FILE_NAME}: Search query error: ${err.message}`); - reject(err); - } else { - logger.debug(`${FILE_NAME}: Search query returned ${rows ? rows.length : 0} results`); - resolve(rows || []); - } - }); - }); - - logger.debug(`${FILE_NAME}: Search results page for keyword "${sanitizedKeyword}": ${results.length} matches (page ${Math.floor(offset / limit) + 1})`); - - return { - results: results.map(r => ({ id: r.rule_id, title: r.title })), - totalCount - }; - } catch (error) { - logger.error(`${FILE_NAME}: Error in search operation: ${error.message}`); - logger.debug(`${FILE_NAME}: Search error stack: ${error.stack}`); - return { results: [], totalCount: 0 }; - } finally { - // Make sure we properly close the connection - if (db) { - try { - await new Promise((resolve) => db.close(() => resolve())); - logger.debug(`${FILE_NAME}: Database connection closed after search operation`); - } catch (closeError) { - logger.error(`${FILE_NAME}: Error closing database connection after search: ${closeError.message}`); - } - } - } -} - -/** - * Check if FTS5 virtual table is available - * - * @param {Object} db - Database connection - * @returns {Promise} Whether FTS5 is available - */ -async function checkFtsAvailable(db) { - try { - const result = await new Promise((resolve, reject) => { - db.get("SELECT name FROM sqlite_master WHERE type='table' AND name='rule_search'", (err, row) => { - if (err) { - logger.error(`${FILE_NAME}: Error checking for FTS5 table: ${err.message}`); - reject(err); - } else { - resolve(row !== undefined); - } - }); - }); - - logger.debug(`${FILE_NAME}: FTS5 table availability check: ${result ? 'Available' : 'Not available'}`); - return result; - } catch (error) { - logger.error(`${FILE_NAME}: Error checking FTS availability: ${error.message}`); - return false; - } -} - -/** - * Search for Sigma rules using FTS5 - * Performs a full-text search and returns matching rules with pagination - * - * @param {string} keyword - The keyword to search for - * @param {number} limit - Maximum number of results to return (default: 10) - * @param {number} offset - Number of results to skip (for pagination, default: 0) - * @returns {Promise} Object with results array and total count - */ -async function searchRulesFTS(keyword, limit = 10, offset = 0) { - if (!keyword) { - logger.warn(`${FILE_NAME}: Empty search keyword provided for FTS search`); - return { results: [], totalCount: 0 }; - } - - // Prepare FTS query - add * for prefix matching if not already present - let ftsQuery = keyword.trim(); - if (!ftsQuery.endsWith('*')) { - ftsQuery = `${ftsQuery}*`; - } - - logger.info(`${FILE_NAME}: Performing FTS search with query: "${ftsQuery}" (limit: ${limit}, offset: ${offset})`); - - let db; - try { - db = await getDbConnection(); - logger.debug(`${FILE_NAME}: Database connection established for FTS search`); - - // First get the total count of matching rules - const countQuery = ` - SELECT COUNT(*) as count - FROM rule_search - WHERE rule_search MATCH ? - `; - - const countResult = await new Promise((resolve, reject) => { - db.get(countQuery, [ftsQuery], (err, row) => { - if (err) { - logger.error(`${FILE_NAME}: FTS count query error: ${err.message}`); - reject(err); - } else { - resolve(row || { count: 0 }); - } - }); - }); - - const totalCount = countResult.count; - logger.debug(`${FILE_NAME}: Total matching rules for FTS query "${ftsQuery}": ${totalCount}`); - - // Now get the actual results with pagination - const searchQuery = ` - SELECT rule_id, title - FROM rule_search - WHERE rule_search MATCH ? - ORDER BY rank - LIMIT ? OFFSET ? - `; - - const results = await new Promise((resolve, reject) => { - db.all(searchQuery, [ftsQuery, limit, offset], (err, rows) => { - if (err) { - logger.error(`${FILE_NAME}: FTS search query error: ${err.message}`); - reject(err); - } else { - logger.debug(`${FILE_NAME}: FTS search query returned ${rows ? rows.length : 0} results`); - resolve(rows || []); - } - }); - }); - - logger.debug(`${FILE_NAME}: FTS search results page for query "${ftsQuery}": ${results.length} matches (page ${Math.floor(offset / limit) + 1})`); - - return { - results: results.map(r => ({ id: r.rule_id, title: r.title || r.rule_id })), - totalCount - }; - } catch (error) { - logger.error(`${FILE_NAME}: Error in FTS search operation: ${error.message}`); - logger.debug(`${FILE_NAME}: FTS search error stack: ${error.stack}`); - return { results: [], totalCount: 0 }; - } finally { - if (db) { - try { - await db.close(); - logger.debug(`${FILE_NAME}: Database connection closed after FTS search operation`); - } catch (closeError) { - logger.error(`${FILE_NAME}: Error closing database connection after FTS search: ${closeError.message}`); - } - } - } -} - -/** - * Search for Sigma rules using complex query conditions - * Supports filtering by multiple attributes like title, logsource, tags, etc. - * - * @param {Object} parsedQuery - The parsed query object containing conditions and operator - * @param {number} limit - Maximum number of results to return - * @param {number} offset - Number of results to skip (for pagination) - * @returns {Promise} Object with results array and total count - */ -async function searchRulesComplex(parsedQuery, limit = 10, offset = 0) { - if (!parsedQuery || !parsedQuery.valid) { - logger.warn(`${FILE_NAME}: Invalid query object provided`); - return { results: [], totalCount: 0 }; - } - - logger.info(`${FILE_NAME}: Performing complex search with ${parsedQuery.conditions.length} conditions (limit: ${limit}, offset: ${offset})`); - - let db; - // Declare this at function scope so it's available in the finally block - let usingFts = false; - - try { - db = await getDbConnection(); - logger.debug(`${FILE_NAME}: Database connection established for complex search`); - - // Check if FTS5 is available - const ftsAvailable = await checkFtsAvailable(db); - - if (ftsAvailable) { - logger.debug(`${FILE_NAME}: Using FTS5 for complex search`); - // Set flag that we're using FTS - usingFts = true; - // Pass db connection to searchRulesComplexFTS and let that function manage it - const results = await searchRulesComplexFTS(parsedQuery, limit, offset, db); - return results; - } - - logger.debug(`${FILE_NAME}: FTS5 not available, using legacy complex search method`); - - // Build the SQL query based on the conditions - const { sqlQuery, sqlCountQuery, params } = buildComplexSqlQuery(parsedQuery, limit, offset); - - logger.debug(`${FILE_NAME}: Executing complex search SQL: ${sqlQuery}`); - logger.debug(`${FILE_NAME}: Query parameters: ${JSON.stringify(params)}`); - - // First get the total count of matching results - const countResult = await new Promise((resolve, reject) => { - db.get(sqlCountQuery, params.slice(0, params.length - 2), (err, row) => { - if (err) { - logger.error(`${FILE_NAME}: Complex search count query error: ${err.message}`); - reject(err); - } else { - resolve(row || { count: 0 }); - } - }); - }); - - const totalCount = countResult.count; - logger.debug(`${FILE_NAME}: Total matching rules for complex query: ${totalCount}`); - - // Now get the actual results with pagination - const results = await new Promise((resolve, reject) => { - db.all(sqlQuery, params, (err, rows) => { - if (err) { - logger.error(`${FILE_NAME}: Complex search query error: ${err.message}`); - reject(err); - } else { - logger.debug(`${FILE_NAME}: Complex search query returned ${rows ? rows.length : 0} results`); - resolve(rows || []); - } - }); - }); - - // Format the results - const formattedResults = results.map(r => ({ - id: r.rule_id, - title: r.title || r.rule_id - })); - - logger.debug(`${FILE_NAME}: Returning ${formattedResults.length} results for complex search`); - - return { - results: formattedResults, - totalCount - }; - } catch (error) { - logger.error(`${FILE_NAME}: Error in complex search operation: ${error.message}`); - logger.debug(`${FILE_NAME}: Complex search error stack: ${error.stack}`); - return { results: [], totalCount: 0 }; - } finally { - // IMPORTANT: Only close the db connection if we're not using FTS - // When using FTS, let searchRulesComplexFTS manage the connection - if (db && !usingFts) { - try { - await new Promise((resolve) => db.close(() => resolve())); - logger.debug(`${FILE_NAME}: Database connection closed after complex search operation`); - } catch (closeError) { - logger.error(`${FILE_NAME}: Error closing database after complex search: ${closeError.message}`); - } - } - } -} - -/** - * Search for Sigma rules using complex query conditions with FTS5 - * Uses the FTS5 virtual table for faster text searching - * - * @param {Object} parsedQuery - The parsed query object - * @param {number} limit - Maximum number of results to return - * @param {number} offset - Number of results to skip (for pagination) - * @param {Object} providedDb - Database connection (optional, will create one if not provided) - * @returns {Promise} Object with results array and total count - */ -async function searchRulesComplexFTS(parsedQuery, limit = 10, offset = 0, providedDb = null) { - if (!parsedQuery || !parsedQuery.valid) { - logger.warn(`${FILE_NAME}: Invalid query object provided for FTS complex search`); - return { results: [], totalCount: 0 }; - } - - logger.info(`${FILE_NAME}: Performing complex FTS search with ${parsedQuery.conditions.length} conditions`); - - let db; - let shouldCloseDb = false; - - try { - // Use provided db connection or create a new one - if (providedDb) { - db = providedDb; - } else { - db = await getDbConnection(); - shouldCloseDb = true; - logger.debug(`${FILE_NAME}: Created new database connection for complex FTS search`); - } - - // Build FTS query from conditions - const { ftsQuery, whereClause, params } = buildComplexFtsQuery(parsedQuery); - - logger.debug(`${FILE_NAME}: FTS query: "${ftsQuery}", additional where: ${whereClause ? whereClause : 'none'}`); - logger.debug(`${FILE_NAME}: Query parameters: ${JSON.stringify(params)}`); - - // Build count query - let countQuery; - let countParams; - - if (whereClause) { - countQuery = ` - SELECT COUNT(*) as count - FROM rule_search - WHERE rule_search MATCH ? - AND ${whereClause} - `; - countParams = [ftsQuery, ...params]; - } else { - countQuery = ` - SELECT COUNT(*) as count - FROM rule_search - WHERE rule_search MATCH ? - `; - countParams = [ftsQuery]; - } - - // Get total count - const countResult = await new Promise((resolve, reject) => { - db.get(countQuery, countParams, (err, row) => { - if (err) { - logger.error(`${FILE_NAME}: Complex FTS count query error: ${err.message}`); - reject(err); - } else { - resolve(row || { count: 0 }); - } - }); - }); - - const totalCount = countResult.count; - logger.debug(`${FILE_NAME}: Total matching rules for complex FTS query: ${totalCount}`); - - // Build results query with pagination - let searchQuery; - let searchParams; - - if (whereClause) { - searchQuery = ` - SELECT rule_id, title - FROM rule_search - WHERE rule_search MATCH ? - AND ${whereClause} - ORDER BY rank - LIMIT ? OFFSET ? - `; - searchParams = [ftsQuery, ...params, limit, offset]; - } else { - searchQuery = ` - SELECT rule_id, title - FROM rule_search - WHERE rule_search MATCH ? - ORDER BY rank - LIMIT ? OFFSET ? - `; - searchParams = [ftsQuery, limit, offset]; - } - - // Get paginated results - const results = await new Promise((resolve, reject) => { - db.all(searchQuery, searchParams, (err, rows) => { - if (err) { - logger.error(`${FILE_NAME}: Complex FTS search query error: ${err.message}`); - reject(err); - } else { - logger.debug(`${FILE_NAME}: Complex FTS search query returned ${rows ? rows.length : 0} results`); - resolve(rows || []); - } - }); - }); - - // Format the results - const formattedResults = results.map(r => ({ - id: r.rule_id, - title: r.title || r.rule_id - })); - - logger.debug(`${FILE_NAME}: Returning ${formattedResults.length} results for complex FTS search`); - - return { - results: formattedResults, - totalCount - }; - } catch (error) { - logger.error(`${FILE_NAME}: Error in complex FTS search operation: ${error.message}`); - logger.debug(`${FILE_NAME}: Complex FTS search error stack: ${error.stack}`); - return { results: [], totalCount: 0 }; - } finally { - // Only close the database if we created it AND we're not in the middle of a transaction - if (db && shouldCloseDb) { - try { - await db.close(); - logger.debug(`${FILE_NAME}: Database connection closed after complex FTS search`); - } catch (closeError) { - logger.error(`${FILE_NAME}: Error closing database after complex FTS search: ${closeError.message}`); - } - } - } -} - -/** - * Build FTS query and WHERE clause from parsed query conditions - * - * @param {Object} parsedQuery - The parsed query object - * @returns {Object} Object with FTS query, additional WHERE clause, and parameters - */ -function buildComplexFtsQuery(parsedQuery) { - const { conditions, operator } = parsedQuery; - - // Separate text search conditions from other conditions - const textConditions = []; - const nonTextConditions = []; - - for (const condition of conditions) { - switch (condition.field) { - case 'title': - case 'description': - case 'author': - case 'tags': - case 'keyword': - // These can be handled by FTS directly - textConditions.push(condition); - break; - default: - // These need additional WHERE clauses - nonTextConditions.push(condition); - break; - } - } - - // Build FTS MATCH query - let ftsQueryParts = []; - - for (const condition of textConditions) { - let fieldPrefix = ''; - - // Add field-specific prefix if available - if (condition.field !== 'keyword') { - fieldPrefix = `${condition.field}:`; - } - - // Add wildcard for partial matching if not already present - let value = condition.value.trim(); - if (!value.endsWith('*')) { - value = `${value}*`; - } - - ftsQueryParts.push(`${fieldPrefix}${value}`); - } - - // If no text conditions, use a match-all query - const ftsQuery = ftsQueryParts.length > 0 - ? ftsQueryParts.join(operator === 'AND' ? ' AND ' : ' OR ') - : '*'; - - // Build additional WHERE clauses for non-text conditions - let whereClauseParts = []; - const params = []; - - for (const condition of nonTextConditions) { - switch (condition.field) { - case 'date': - const dateOperator = condition.operator === 'after' ? '>' : - condition.operator === 'before' ? '<' : '='; - whereClauseParts.push(`date ${dateOperator} date(?)`); - params.push(condition.value); - break; - - case 'level': - whereClauseParts.push(`level = ?`); - params.push(condition.value); - break; - - case 'logsource': - whereClauseParts.push(`logsource LIKE ?`); - params.push(`%${condition.subfield}%${condition.value}%`); - break; - - case 'id': - whereClauseParts.push(`rule_id = ?`); - params.push(condition.value); - break; - } - } - - // Combine WHERE clauses - const whereClause = whereClauseParts.length > 0 - ? whereClauseParts.join(operator === 'AND' ? ' AND ' : ' OR ') - : ''; - - return { ftsQuery, whereClause, params }; -} - -/** - * Build the SQL query for complex search based on parsed conditions - * - * @param {Object} parsedQuery - The parsed query object - * @param {number} limit - Results limit - * @param {number} offset - Results offset - * @returns {Object} Object with SQL query, count query, and parameters - */ -function buildComplexSqlQuery(parsedQuery, limit, offset) { - const { conditions, operator } = parsedQuery; - const params = []; - - // Start building the primary table selection - let sqlSelectPart = ` - SELECT DISTINCT r.id as rule_id, - (SELECT param_value FROM rule_parameters WHERE rule_id = r.id AND param_name = 'title' LIMIT 1) as title - FROM sigma_rules r - `; - - // Build WHERE clause based on conditions - let whereClauses = []; - let joinIdx = 0; - - for (const condition of conditions) { - let whereClause = ''; - - switch (condition.field) { - case 'title': - joinIdx++; - whereClause = `EXISTS ( - SELECT 1 FROM rule_parameters p${joinIdx} - WHERE p${joinIdx}.rule_id = r.id - AND p${joinIdx}.param_name = 'title' - AND INSTR(LOWER(p${joinIdx}.param_value), LOWER(?)) > 0 - )`; - params.push(condition.value); - break; - - case 'description': - joinIdx++; - whereClause = `EXISTS ( - SELECT 1 FROM rule_parameters p${joinIdx} - WHERE p${joinIdx}.rule_id = r.id - AND p${joinIdx}.param_name = 'description' - AND INSTR(LOWER(p${joinIdx}.param_value), LOWER(?)) > 0 - )`; - params.push(condition.value); - break; - - case 'logsource': - joinIdx++; - whereClause = `EXISTS ( - SELECT 1 FROM rule_parameters p${joinIdx} - WHERE p${joinIdx}.rule_id = r.id - AND p${joinIdx}.param_name = 'logsource' - AND INSTR(LOWER(p${joinIdx}.param_value), LOWER('"${condition.subfield}":"${condition.value}"')) > 0 - )`; - break; - - case 'tags': - joinIdx++; - whereClause = `EXISTS ( - SELECT 1 FROM rule_parameters p${joinIdx} - WHERE p${joinIdx}.rule_id = r.id - AND p${joinIdx}.param_name = 'tags' - AND INSTR(LOWER(p${joinIdx}.param_value), LOWER(?)) > 0 - )`; - params.push(condition.value); - break; - - case 'date': - joinIdx++; - const dateOperator = condition.operator === 'after' ? '>' : - condition.operator === 'before' ? '<' : '='; - whereClause = `r.date ${dateOperator} date(?)`; - params.push(condition.value); - break; - - case 'author': - joinIdx++; - whereClause = `EXISTS ( - SELECT 1 FROM rule_parameters p${joinIdx} - WHERE p${joinIdx}.rule_id = r.id - AND p${joinIdx}.param_name = 'author' - AND INSTR(LOWER(p${joinIdx}.param_value), LOWER(?)) > 0 - )`; - params.push(condition.value); - break; - - case 'level': - joinIdx++; - whereClause = `EXISTS ( - SELECT 1 FROM rule_parameters p${joinIdx} - WHERE p${joinIdx}.rule_id = r.id - AND p${joinIdx}.param_name = 'level' - AND LOWER(p${joinIdx}.param_value) = LOWER(?) - )`; - params.push(condition.value); - break; - - case 'id': - whereClause = `LOWER(r.id) = LOWER(?)`; - params.push(condition.value); - break; - - case 'keyword': - default: - // Default to searching in title - joinIdx++; - whereClause = `EXISTS ( - SELECT 1 FROM rule_parameters p${joinIdx} - WHERE p${joinIdx}.rule_id = r.id - AND p${joinIdx}.param_name = 'title' - AND INSTR(LOWER(p${joinIdx}.param_value), LOWER(?)) > 0 - )`; - params.push(condition.value); - break; - } - - if (whereClause) { - whereClauses.push(whereClause); - } - } - - // Combine the WHERE clauses with the appropriate operator - let whereStatement = ''; - if (whereClauses.length > 0) { - const combiner = operator === 'AND' ? ' AND ' : ' OR '; - whereStatement = `WHERE ${whereClauses.join(combiner)}`; - } - - // Complete queries - const sqlQuery = ` - ${sqlSelectPart} - ${whereStatement} - ORDER BY rule_id - LIMIT ? OFFSET ? - `; - - const sqlCountQuery = ` - SELECT COUNT(DISTINCT r.id) as count - FROM sigma_rules r - ${whereStatement} - `; - - // Add pagination parameters - params.push(limit); - params.push(offset); - - return { sqlQuery, sqlCountQuery, params }; -} - -/** - * Debug function to retrieve detailed information about a rule's content - * Useful for diagnosing issues with rule retrieval and content parsing - * - * @param {string} ruleId - The ID of the rule to debug - * @returns {Promise} Object containing debug information or null on error - */ -async function debugRuleContent(ruleId) { - if (!ruleId) { - logger.warn(`${FILE_NAME}: Cannot debug rule: Missing rule ID`); - return null; - } - - let db; - try { - db = await getDbConnection(); - - const absolutePath = path.resolve(DB_PATH); - logger.debug(`${FILE_NAME}: Debug function connecting to DB at path: ${absolutePath}`); - - // Get raw rule record - const rule = await db.get('SELECT id, file_path, length(content) as content_length, typeof(content) as content_type FROM sigma_rules WHERE id = ?', [ruleId]); - - if (!rule) { - logger.warn(`${FILE_NAME}: Rule with ID ${ruleId} not found during debug operation`); - return { error: 'Rule not found', ruleId }; - } - - // Return just the rule information without the undefined variables - return { - rule, - ruleId - }; - } catch (error) { - logger.error(`${FILE_NAME}: Debug error for rule ${ruleId}: ${error.message}`); - logger.debug(`${FILE_NAME}: Debug error stack: ${error.stack}`); - return { - error: error.message, - stack: error.stack, - ruleId - }; - } finally { - if (db) { - try { - await db.close(); - logger.debug(`${FILE_NAME}: Database connection closed after debug operation`); - } catch (closeError) { - logger.warn(`${FILE_NAME}: Error closing database after debug: ${closeError.message}`); - } - } - } -} - -/** - * Get the raw YAML content of a Sigma rule - * Retrieves the content field from the database which should contain YAML - * - * @param {string} ruleId - The ID of the rule - * @returns {Promise} Object with success flag and content or error message - */ -async function getRuleYamlContent(ruleId) { - if (!ruleId) { - logger.warn(`${FILE_NAME}: Cannot get YAML content: Missing rule ID`); - return { success: false, message: 'Missing rule ID' }; - } - - let db; - try { - logger.info(`${FILE_NAME}: Fetching YAML content for rule: ${ruleId}`); - logger.debug(`${FILE_NAME}: Rule ID type: ${typeof ruleId}, length: ${ruleId.length}`); - - db = await getDbConnection(); - logger.debug(`${FILE_NAME}: Connected to database for YAML retrieval`); - - // Debug query before execution - const debugResult = await db.get('SELECT id, typeof(content) as content_type, length(content) as content_length FROM sigma_rules WHERE id = ?', [ruleId]); - logger.debug(`${FILE_NAME}: Debug query result: ${JSON.stringify(debugResult || 'not found')}`); - - if (!debugResult) { - logger.warn(`${FILE_NAME}: Rule with ID ${ruleId} not found in debug query`); - return { success: false, message: 'Rule not found' }; - } - - // Get actual content - const rule = await new Promise((resolve, reject) => { - db.get('SELECT content FROM sigma_rules WHERE id = ?', [ruleId], (err, row) => { - if (err) { - logger.error(`${FILE_NAME}: Content query error: ${err.message}`); - reject(err); - } else { - resolve(row || null); - } - }); - }); - logger.debug(`${FILE_NAME}: Content query result for ${ruleId}: ${rule ? 'Found' : 'Not found'}`); - - if (!rule) { - logger.warn(`${FILE_NAME}: Rule with ID ${ruleId} not found in content query`); - return { success: false, message: 'Rule not found' }; - } - - if (!rule.content) { - logger.warn(`${FILE_NAME}: Rule with ID ${ruleId} found but has no content`); - return { success: false, message: 'Rule found but content is empty' }; - } - - logger.debug(`${FILE_NAME}: Content retrieved successfully for ${ruleId}, type: ${typeof rule.content}, length: ${rule.content.length}`); - - return { success: true, content: rule.content }; - } catch (error) { - logger.error(`${FILE_NAME}: Error retrieving YAML content for ${ruleId}: ${error.message}`); - logger.debug(`${FILE_NAME}: YAML retrieval error stack: ${error.stack}`); - return { success: false, message: `Error retrieving YAML: ${error.message}` }; - } finally { - if (db) { - try { - await db.close(); - logger.debug(`${FILE_NAME}: Database connection closed after YAML retrieval`); - } catch (closeError) { - logger.warn(`${FILE_NAME}: Error closing database after YAML retrieval: ${closeError.message}`); - } - } - } -} - -/** - * Get statistics about Sigma rules in the database - * Collects counts, categories, and other aggregate information - * - * @returns {Promise} Object with various statistics about the rules - */ -async function getStatsFromDatabase() { - let db; - try { - db = await getDbConnection(); - logger.debug(`${FILE_NAME}: Connected to database for statistics`); - - // Get total rule count - const totalRulesResult = await new Promise((resolve, reject) => { - db.get('SELECT COUNT(*) as count FROM sigma_rules', (err, row) => { - if (err) reject(err); - else resolve(row); - }); - }); - const totalRules = totalRulesResult.count; - - // Get last update time - const lastUpdateResult = await new Promise((resolve, reject) => { - db.get('SELECT MAX(date) as last_update FROM sigma_rules', (err, row) => { - if (err) reject(err); - else resolve(row); - }); - }); - const lastUpdate = lastUpdateResult.last_update; - - // Get rules by log source count (Windows, Linux, macOS) - const windowsRulesResult = await new Promise((resolve, reject) => { - db.get(` - SELECT COUNT(DISTINCT rule_id) as count - FROM rule_parameters - WHERE param_name = 'logsource' AND - param_value LIKE '%"product":"windows"%'`, - (err, row) => { - if (err) reject(err); - else resolve(row); - }); - }); - const windowsRules = windowsRulesResult.count || 0; - - const linuxRulesResult = await new Promise((resolve, reject) => { - db.get(` - SELECT COUNT(DISTINCT rule_id) as count - FROM rule_parameters - WHERE param_name = 'logsource' AND - param_value LIKE '%"product":"linux"%'`, - (err, row) => { - if (err) reject(err); - else resolve(row); - }); - }); - const linuxRules = linuxRulesResult.count || 0; - - const macosRulesResult = await new Promise((resolve, reject) => { - db.get(` - SELECT COUNT(DISTINCT rule_id) as count - FROM rule_parameters - WHERE param_name = 'logsource' AND - param_value LIKE '%"product":"macos"%'`, - (err, row) => { - if (err) reject(err); - else resolve(row); - }); - }); - const macosRules = macosRulesResult.count || 0; - - // Get rules by severity level - const severityStats = await new Promise((resolve, reject) => { - db.all(` - SELECT param_value AS level, COUNT(DISTINCT rule_id) as count - FROM rule_parameters - WHERE param_name = 'level' - GROUP BY param_value - ORDER BY - CASE - WHEN param_value = 'critical' THEN 1 - WHEN param_value = 'high' THEN 2 - WHEN param_value = 'medium' THEN 3 - WHEN param_value = 'low' THEN 4 - WHEN param_value = 'informational' THEN 5 - ELSE 6 - END`, - (err, rows) => { - if (err) reject(err); - else resolve(rows); - }); - }); - - // Get top 5 rule authors - const topAuthors = await new Promise((resolve, reject) => { - db.all(` - SELECT param_value AS author, COUNT(*) as count - FROM rule_parameters - WHERE param_name = 'author' - GROUP BY param_value - ORDER BY count DESC - LIMIT 5`, - (err, rows) => { - if (err) reject(err); - else resolve(rows); - }); - }); - - // Get empty content count (rules with missing YAML) - const emptyContentResult = await new Promise((resolve, reject) => { - db.get(` - SELECT COUNT(*) as count - FROM sigma_rules - WHERE content IS NULL OR content = ''`, - (err, row) => { - if (err) reject(err); - else resolve(row); - }); - }); - const emptyContentCount = emptyContentResult.count; - - // Get MITRE ATT&CK tactics statistics - const mitreStats = await new Promise((resolve, reject) => { - db.all(` - SELECT param_value AS tag, COUNT(DISTINCT rule_id) as count - FROM rule_parameters - WHERE param_name = 'tags' AND param_value LIKE 'attack.%' - GROUP BY param_value - ORDER BY count DESC - LIMIT 10`, - (err, rows) => { - if (err) reject(err); - else resolve(rows); - }); - }); - - // Format MITRE tactics for display - const formattedMitreTactics = mitreStats.map(item => { - const tactic = item.tag.substring(7); // Remove 'attack.' prefix - return { - tactic: tactic, - count: item.count - }; - }); - - // Compile all statistics - const stats = { - totalRules, - lastUpdate, - operatingSystems: { - windows: windowsRules, - linux: linuxRules, - macos: macosRules, - other: totalRules - (windowsRules + linuxRules + macosRules) - }, - severityLevels: severityStats.map(s => ({ level: s.level, count: s.count })), - topAuthors: topAuthors.map(a => ({ name: a.author, count: a.count })), - databaseHealth: { - emptyContentCount, - contentPercentage: totalRules > 0 ? Math.round(((totalRules - emptyContentCount) / totalRules) * 100) : 0 - }, - mitreTactics: formattedMitreTactics - }; - - return { - success: true, - stats - }; - } catch (error) { - logger.error(`${FILE_NAME}: Error retrieving statistics: ${error.message}`); - logger.debug(`${FILE_NAME}: Error stack: ${error.stack}`); - return { - success: false, - message: `Error retrieving statistics: ${error.message}` - }; - } finally { - if (db) { - try { - await new Promise((resolve) => db.close(() => resolve())); - logger.debug(`${FILE_NAME}: Database connection closed after statistics retrieval`); - } catch (closeError) { - logger.warn(`${FILE_NAME}: Error closing database: ${closeError.message}`); - } - } - } -} - -module.exports = { - getAllRuleIds, - findRuleById, - searchRules, - searchRulesFTS, - searchRulesComplex, - searchRulesComplexFTS, - debugRuleContent, - getRuleYamlContent, - getStatsFromDatabase, - checkFtsAvailable -}; \ No newline at end of file -- 2.39.5 From 519c87fb049a7625fb72dd0dc60b88cfe7355c6d Mon Sep 17 00:00:00 2001 From: Charlotte Croce Date: Fri, 18 Apr 2025 17:15:41 -0400 Subject: [PATCH 08/15] create CLI and sigma stats function in CLI --- fylgja-cli | 2 + fylgja-cli.md | 59 +++ package-lock.json | 65 ++- package.json | 7 +- src/fylgja-cli.js | 509 ++++++++++++++++++++++ src/handlers/sigma/sigma_stats_handler.js | 5 +- src/services/sigma/sigma_stats_service.js | 16 +- src/utils/cli_formatters.js | 171 ++++++++ src/utils/logger.js | 61 ++- 9 files changed, 849 insertions(+), 46 deletions(-) create mode 100755 fylgja-cli create mode 100644 fylgja-cli.md create mode 100644 src/fylgja-cli.js create mode 100644 src/utils/cli_formatters.js diff --git a/fylgja-cli b/fylgja-cli new file mode 100755 index 0000000..0ac5b97 --- /dev/null +++ b/fylgja-cli @@ -0,0 +1,2 @@ +#!/bin/bash +node "$(dirname "$0")/src/fylgja-cli.js" "$@" diff --git a/fylgja-cli.md b/fylgja-cli.md new file mode 100644 index 0000000..2b5a7f5 --- /dev/null +++ b/fylgja-cli.md @@ -0,0 +1,59 @@ +# Fylgja CLI Interface + +The Fylgja CLI provides an interactive command-line interface for managing SIEM rules, similar to MySQL's CLI. + +## Usage + +Start the CLI interface: + +```bash +npm run cli +``` + +Or use the direct launcher: + +```bash +./fylgja-cli +``` + +## Features + +- **Interactive Prompt**: MySQL-style prompt with command history +- **Tab Completion**: Press Tab to auto-complete commands +- **Command History**: Use Up/Down arrows to navigate previous commands +- **Formatted Output**: Table-based output formats for different commands +- **Color Coding**: Visual indicators for severity levels and result types + +## Available Commands + +### Basic Commands + +``` +search Search for Sigma rules by keyword +details Get details about a specific Sigma rule +stats Get statistics about Sigma rules database +help Display help information +exit/quit Exit the CLI +clear Clear the terminal screen +``` + +### Advanced Search Commands + +``` +search sigma rules where title contains "ransomware" +find rules where tags include privilege_escalation +search rules where logsource.category == "process_creation" +find rules where modified after 2024-01-01 +``` + +## Examples + +``` +fylgja> search rules where level is "high" +fylgja> details 5f35f6c7-80a7-4ca0-a41f-31e8ac557233 +fylgja> stats +``` + +## Integration with Slack Bot + +The CLI interface uses the same command parsing and execution logic as the Slack bot, ensuring consistency across interfaces. \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index 4cb0601..b1c7c87 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10,10 +10,13 @@ "license": "ISC", "dependencies": { "@slack/bolt": "^4.2.1", + "axios": "^1.6.7", + "chalk": "^5.4.1", "dotenv": "^16.4.7", "express": "^5.1.0", "glob": "^8.1.0", "js-yaml": "^4.1.0", + "readline": "^1.3.0", "sqlite3": "^5.1.7" }, "devDependencies": { @@ -659,35 +662,17 @@ } }, "node_modules/chalk": { - "version": "4.1.2", - "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", - "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", - "dev": true, + "version": "5.4.1", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-5.4.1.tgz", + "integrity": "sha512-zgVZuo2WcZgfUEmsn6eO3kINexW8RAE4maiQ8QNs8CtpPCSyMiYsULR3HQYkm3w8FIA3SberyMJMSldGsW+U3w==", "license": "MIT", - "dependencies": { - "ansi-styles": "^4.1.0", - "supports-color": "^7.1.0" - }, "engines": { - "node": ">=10" + "node": "^12.17.0 || ^14.13 || >=16.0.0" }, "funding": { "url": "https://github.com/chalk/chalk?sponsor=1" } }, - "node_modules/chalk/node_modules/supports-color": { - "version": "7.2.0", - "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", - "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", - "dev": true, - "license": "MIT", - "dependencies": { - "has-flag": "^4.0.0" - }, - "engines": { - "node": ">=8" - } - }, "node_modules/chownr": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/chownr/-/chownr-2.0.0.tgz", @@ -797,6 +782,36 @@ "url": "https://github.com/open-cli-tools/concurrently?sponsor=1" } }, + "node_modules/concurrently/node_modules/chalk": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", + "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-styles": "^4.1.0", + "supports-color": "^7.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/chalk?sponsor=1" + } + }, + "node_modules/concurrently/node_modules/chalk/node_modules/supports-color": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", + "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", + "dev": true, + "license": "MIT", + "dependencies": { + "has-flag": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/console-control-strings": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/console-control-strings/-/console-control-strings-1.1.0.tgz", @@ -2441,6 +2456,12 @@ "node": ">= 6" } }, + "node_modules/readline": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/readline/-/readline-1.3.0.tgz", + "integrity": "sha512-k2d6ACCkiNYz222Fs/iNze30rRJ1iIicW7JuX/7/cozvih6YCkFZH+J6mAFDVgv0dRBaAyr4jDqC95R2y4IADg==", + "license": "BSD" + }, "node_modules/require-directory": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", diff --git a/package.json b/package.json index 1cdbd06..283c675 100644 --- a/package.json +++ b/package.json @@ -7,7 +7,8 @@ "ngrok": "ngrok http 3000 --log=stdout --url=tolerant-bull-ideal.ngrok-free.app", "dev": "concurrently \"npm run start\" \"npm run ngrok\"", "update-db": "node src/sigma_db/sigma_db_initialize.js", - "test": "echo \"Error: no test specified\" && exit 1" + "test": "echo \"Error: no test specified\" && exit 1", + "cli": "node src/fylgja-cli.js" }, "keywords": [], "author": "", @@ -16,13 +17,15 @@ "dependencies": { "@slack/bolt": "^4.2.1", "axios": "^1.6.7", + "chalk": "^5.4.1", "dotenv": "^16.4.7", "express": "^5.1.0", "glob": "^8.1.0", "js-yaml": "^4.1.0", + "readline": "^1.3.0", "sqlite3": "^5.1.7" }, "devDependencies": { "concurrently": "^9.1.2" } -} \ No newline at end of file +} diff --git a/src/fylgja-cli.js b/src/fylgja-cli.js new file mode 100644 index 0000000..612665f --- /dev/null +++ b/src/fylgja-cli.js @@ -0,0 +1,509 @@ +/** + * fylgja-cli.js + * + * Interactive CLI interface + */ + +const readline = require('readline'); +// Import chalk with compatibility for both ESM and CommonJS +let chalk; +try { + // First try CommonJS import (chalk v4.x) + chalk = require('chalk'); +} catch (e) { + // If that fails, provide a fallback implementation + chalk = { + blue: (text) => text, + green: (text) => text, + red: (text) => text, + yellow: (text) => text, + cyan: (text) => text, + white: (text) => text, + dim: (text) => text, + hex: () => (text) => text + }; +} + +const { parseCommand } = require('./lang/command_parser'); +const logger = require('./utils/logger'); +const sigmaSearchHandler = require('./handlers/sigma/sigma_search_handler'); +const sigmaDetailsHandler = require('./handlers/sigma/sigma_details_handler'); +const sigmaStatsHandler = require('./handlers/sigma/sigma_stats_handler'); +const sigmaCreateHandler = require('./handlers/sigma/sigma_create_handler'); +const { handleCommand: handleAlerts } = require('./handlers/alerts/alerts_handler'); +const { handleCommand: handleCase } = require('./handlers/case/case_handler'); +const { handleCommand: handleConfig } = require('./handlers/config/config_handler'); +const { handleCommand: handleStats } = require('./handlers/stats/stats_handler'); + +// Import CLI formatters +const { + formatSigmaStats, + formatSigmaSearchResults, + formatSigmaDetails +} = require('./utils/cli_formatters'); + +// Set logger to CLI mode (prevents console output) +logger.setCliMode(true); + +// Try to get version, but provide fallback if package.json can't be found +let version = '1.0.0'; +try { + const packageJson = require('../package.json'); + version = packageJson.version; +} catch (e) { + console.log('Could not load package.json, using default version'); +} + +const FILE_NAME = 'fylgja-cli.js'; + +// ASCII art logo for the CLI +const ASCII_LOGO = ` +░▒▓████████▓▒░▒▓█▓▒░░▒▓█▓▒░▒▓█▓▒░ ░▒▓██████▓▒░ ░▒▓█▓▒░░▒▓██████▓▒░ +░▒▓█▓▒░ ░▒▓█▓▒░░▒▓█▓▒░▒▓█▓▒░ ░▒▓█▓▒░░▒▓█▓▒░ ░▒▓█▓▒░▒▓█▓▒░░▒▓█▓▒░ +░▒▓█▓▒░ ░▒▓█▓▒░░▒▓█▓▒░▒▓█▓▒░ ░▒▓█▓▒░ ░▒▓█▓▒░▒▓█▓▒░░▒▓█▓▒░ +░▒▓██████▓▒░ ░▒▓██████▓▒░░▒▓█▓▒░ ░▒▓█▓▒▒▓███▓▒░ ░▒▓█▓▒░▒▓████████▓▒░ +░▒▓█▓▒░ ░▒▓█▓▒░ ░▒▓█▓▒░ ░▒▓█▓▒░░▒▓█▓▒░▒▓█▓▒░░▒▓█▓▒░▒▓█▓▒░░▒▓█▓▒░ +░▒▓█▓▒░ ░▒▓█▓▒░ ░▒▓█▓▒░ ░▒▓█▓▒░░▒▓█▓▒░▒▓█▓▒░░▒▓█▓▒░▒▓█▓▒░░▒▓█▓▒░ +░▒▓█▓▒░ ░▒▓█▓▒░ ░▒▓████████▓▒░▒▓██████▓▒░ ░▒▓██████▓▒░░▒▓█▓▒░░▒▓█▓▒░ +`; + +// Command history array +let commandHistory = []; +let historyIndex = -1; + +// Create the readline interface +const rl = readline.createInterface({ + input: process.stdin, + output: process.stdout, + completer: completer, + prompt: 'fylgja> ' +}); + +/** + * Command auto-completion function + * @param {string} line Current command line input + * @returns {Array} Array with possible completions and the substring being completed + */ +function completer(line) { + const commands = [ + 'search sigma', + 'details sigma', + 'sigma stats', + 'stats sigma', + 'search sigma rules where title contains', + 'search rules where tags include', + 'search rules where logsource.category ==', + 'search rules where modified after', + 'help', + 'exit', + 'quit', + 'clear' + ]; + + const hits = commands.filter((c) => c.startsWith(line)); + return [hits.length ? hits : commands, line]; +} + +/** + * Format CLI output similar to MySQL + * @param {Object} data The data to format + * @param {string} type The type of data (results, details, stats) + */ +function formatOutput(data, type) { + if (!data) { + console.log('No data returned from the server.'); + return; + } + + switch (type) { + case 'search_results': + console.log('\n+-------+----------------------+------------------+-------------+'); + console.log('| ID | Title | Author | Level |'); + console.log('+-------+----------------------+------------------+-------------+'); + + if (data.results && data.results.length > 0) { + data.results.forEach(rule => { + const id = (rule.id || '').padEnd(5).substring(0, 5); + const title = (rule.title || '').padEnd(20).substring(0, 20); + const author = (rule.author || 'Unknown').padEnd(16).substring(0, 16); + const level = (rule.level || 'medium').padEnd(11).substring(0, 11); + + console.log(`| ${id} | ${title} | ${author} | ${level} |`); + }); + } else { + console.log('| No results found |'); + } + + console.log('+-------+----------------------+------------------+-------------+'); + console.log(`${data.totalCount || 0} rows in set`); + break; + + case 'details': + console.log('\n+----------------------+--------------------------------------------------+'); + console.log('| Field | Value |'); + console.log('+----------------------+--------------------------------------------------+'); + + for (const [key, value] of Object.entries(data)) { + if (typeof value !== 'object' || value === null) { + const formattedKey = key.padEnd(20).substring(0, 20); + const formattedValue = String(value || '').padEnd(48).substring(0, 48); + + console.log(`| ${formattedKey} | ${formattedValue} |`); + } + } + + console.log('+----------------------+--------------------------------------------------+'); + break; + + case 'stats': + console.log('\n+--------------------+---------------+'); + console.log('| Metric | Value |'); + console.log('+--------------------+---------------+'); + + for (const [key, value] of Object.entries(data)) { + const formattedKey = key.padEnd(18).substring(0, 18); + const formattedValue = String(value || '').padEnd(13).substring(0, 13); + + console.log(`| ${formattedKey} | ${formattedValue} |`); + } + + console.log('+--------------------+---------------+'); + break; + + default: + console.log(JSON.stringify(data, null, 2)); + } +} + +/** + * Parse out any basic search keywords from a complexSearch query + * This helps with the search commands that don't quite match the expected format + * @param {string} input The complex search query + * @returns {string} Extracted keywords + */ +function extractSearchKeywords(input) { + if (!input) return ''; + + // Try to extract keywords from common patterns + if (input.includes('title contains')) { + const match = input.match(/title\s+contains\s+["']([^"']+)["']/i); + if (match) return match[1]; + } + + if (input.includes('tags include')) { + const match = input.match(/tags\s+include\s+(\S+)/i); + if (match) return match[1]; + } + + // Default - just return the input as is + return input; +} + +/** + * Process a command from the CLI + * @param {string} input User input command + */ +async function processCommand(input) { + try { + // Skip empty commands + if (!input.trim()) { + rl.prompt(); + return; + } + + // Special CLI commands + if (input.trim().toLowerCase() === 'exit' || input.trim().toLowerCase() === 'quit') { + console.log('Goodbye!'); + rl.close(); + process.exit(0); + } + + if (input.trim().toLowerCase() === 'clear') { + console.clear(); + rl.prompt(); + return; + } + + // Special case for simple search + if (input.trim().match(/^search\s+sigma\s+(.+)$/i)) { + const keyword = input.trim().match(/^search\s+sigma\s+(.+)$/i)[1]; + + // Add to command history + commandHistory.push(input); + historyIndex = commandHistory.length; + + // Create fake command object + const command = { + text: keyword, + user_id: 'cli_user', + user_name: 'cli_user', + command: '/fylgja', + channel_id: 'cli', + channel_name: 'cli' + }; + + // Create custom respond function + const respond = createRespondFunction('search', 'sigma', [keyword]); + + console.log(`Executing: module=sigma, action=search, params=[${keyword}]`); + + try { + await sigmaSearchHandler.handleCommand(command, respond); + } catch (error) { + console.error(`Error: ${error.message}`); + logger.error(`${FILE_NAME}: Command execution error: ${error.message}`); + logger.debug(`${FILE_NAME}: Error stack: ${error.stack}`); + rl.prompt(); + } + + return; + } + + // Add to command history + commandHistory.push(input); + historyIndex = commandHistory.length; + + // Parse command using existing parser + const parsedCommand = await parseCommand(input); + + if (!parsedCommand.success) { + console.log(parsedCommand.message || "Command not recognized. Type 'help' for usage."); + rl.prompt(); + return; + } + + // Extract the command details + const { action, module, params } = parsedCommand.command; + + // Only show execution info to the user, not sending to logger + console.log(`Executing: module=${module}, action=${action}, params=[${params}]`); + + // Create fake command object similar to Slack's + const command = { + text: Array.isArray(params) && params.length > 0 ? params[0] : input, + user_id: 'cli_user', + user_name: 'cli_user', + command: '/fylgja', + channel_id: 'cli', + channel_name: 'cli' + }; + + // Special handling for complexSearch to extract keywords + if (action === 'complexSearch' && module === 'sigma' && params.length > 0) { + // Try to extract keywords from complex queries + const searchTerms = extractSearchKeywords(params[0]); + command.text = searchTerms || params[0]; + } + + // Create custom respond function for CLI + const respond = createRespondFunction(action, module, params); + + try { + switch (module) { + case 'sigma': + switch (action) { + case 'search': + await sigmaSearchHandler.handleCommand(command, respond); + break; + + case 'complexSearch': + await sigmaSearchHandler.handleComplexSearch(command, respond); + break; + + case 'details': + await sigmaDetailsHandler.handleCommand(command, respond); + break; + + case 'stats': + await sigmaStatsHandler.handleCommand(command, respond); + break; + + case 'create': + await sigmaCreateHandler.handleCommand(command, respond); + break; + + default: + console.log(`Unknown Sigma action: ${action}`); + rl.prompt(); + } + break; + + case 'alerts': + await handleAlerts(command, respond); + break; + + case 'case': + await handleCase(command, respond); + break; + + case 'config': + await handleConfig(command, respond); + break; + + case 'stats': + await handleStats(command, respond); + break; + + case 'help': + displayHelp(); + rl.prompt(); + break; + + default: + console.log(`Unknown module: ${module}`); + rl.prompt(); + } + } catch (error) { + console.error(`Error: ${error.message}`); + // Log to file but not console + logger.error(`${FILE_NAME}: Command execution error: ${error.message}`); + logger.debug(`${FILE_NAME}: Error stack: ${error.stack}`); + rl.prompt(); + } + } catch (error) { + console.error(`Fatal error: ${error.message}`); + // Log to file but not console + logger.error(`${FILE_NAME}: Fatal error: ${error.message}`); + logger.debug(`${FILE_NAME}: Error stack: ${error.stack}`); + rl.prompt(); + } +} + +/** + * Create a custom respond function for handling results + * @param {string} action The action being performed + * @param {string} module The module being used + * @param {Array} params The parameters for the action + * @returns {Function} A respond function for handling results + */ +function createRespondFunction(action, module, params) { + return async (response) => { + if (typeof response === 'string') { + console.log(response); + rl.prompt(); + return; + } + + // First check for the responseData property (directly from service) + if (response.responseData) { + // Format the data using the appropriate formatter + if (module === 'sigma') { + let formattedData; + + if (action === 'search' || action === 'complexSearch') { + formattedData = formatSigmaSearchResults(response.responseData); + formatOutput(formattedData, 'search_results'); + } else if (action === 'details') { + formattedData = formatSigmaDetails(response.responseData); + formatOutput(formattedData, 'details'); + } else if (action === 'stats') { + formattedData = formatSigmaStats(response.responseData); + formatOutput(formattedData, 'stats'); + } else { + console.log(JSON.stringify(response.responseData, null, 2)); + } + } else { + // For other modules, just display the JSON + console.log(JSON.stringify(response.responseData, null, 2)); + } + } + // Fallback for text-only responses + else if (response.text) { + console.log(response.text); + } else { + console.log('Command completed successfully.'); + } + + rl.prompt(); + }; +} + +/** + * Display help text + */ +function displayHelp() { + const helpText = ` +Fylgja CLI Help + +Basic Sigma Commands: +- search sigma - Search for Sigma rules by keyword +- details sigma - Get details about a specific Sigma rule +- sigma stats - Get statistics about Sigma rules database + +Advanced Sigma Search Commands: +- search sigma rules where title contains "ransomware" - Search by title +- search sigma rules where tags include privilege_escalation - Search by tags +- search sigma rules where logsource.category == "process_creation" - Search by log source +- search sigma rules where modified after 2024-01-01 - Search by modification date + + +- exit or quit - Exit the CLI +- clear - Clear the terminal screen +- help - Display this help text + `; + + console.log(helpText); +} + +/** + * Start the CLI application + */ +function startCLI() { + console.log(ASCII_LOGO); + console.log(`Fylgja CLI v${version} - Interactive SIEM Management Tool`); + console.log(`Type 'help' for usage information or 'exit' to quit\n`); + + // Set up key bindings for history navigation + rl._writeToOutput = function _writeToOutput(stringToWrite) { + if (stringToWrite === '\\u001b[A' || stringToWrite === '\\u001b[B') { + // Don't output control characters for up/down arrows + return; + } + rl.output.write(stringToWrite); + }; + + // Set up key listeners for history + rl.input.on('keypress', (char, key) => { + if (key && key.name === 'up') { + if (historyIndex > 0) { + historyIndex--; + rl.line = commandHistory[historyIndex]; + rl.cursor = rl.line.length; + rl._refreshLine(); + } + } else if (key && key.name === 'down') { + if (historyIndex < commandHistory.length - 1) { + historyIndex++; + rl.line = commandHistory[historyIndex]; + rl.cursor = rl.line.length; + rl._refreshLine(); + } else if (historyIndex === commandHistory.length - 1) { + historyIndex = commandHistory.length; + rl.line = ''; + rl.cursor = 0; + rl._refreshLine(); + } + } + }); + + rl.prompt(); + + rl.on('line', async (line) => { + await processCommand(line.trim()); + }); + + rl.on('close', () => { + console.log('Goodbye!'); + process.exit(0); + }); +} + +// Check if running directly +if (require.main === module) { + startCLI(); +} else { + // Export functions for integration with main app + module.exports = { + startCLI + }; +} \ No newline at end of file diff --git a/src/handlers/sigma/sigma_stats_handler.js b/src/handlers/sigma/sigma_stats_handler.js index 5b49a63..d92e126 100644 --- a/src/handlers/sigma/sigma_stats_handler.js +++ b/src/handlers/sigma/sigma_stats_handler.js @@ -39,7 +39,7 @@ const handleCommand = async (command, respond) => { return; } - // Generate blocks for displaying statistics + // For Slack responses, generate Block Kit blocks let blocks; try { blocks = getStatsBlocks(statsResult.stats); @@ -51,9 +51,10 @@ const handleCommand = async (command, respond) => { return; } - // Return the response + // Return the response with both blocks for Slack and responseData for CLI await respond({ blocks: blocks, + responseData: statsResult.stats, // Include raw data for CLI response_type: 'in_channel' }); } catch (error) { diff --git a/src/services/sigma/sigma_stats_service.js b/src/services/sigma/sigma_stats_service.js index 0bee7a0..20cac15 100644 --- a/src/services/sigma/sigma_stats_service.js +++ b/src/services/sigma/sigma_stats_service.js @@ -31,11 +31,25 @@ async function getSigmaStats() { }; } + // Format the data in a consistent structure for both CLI and Slack + const formattedStats = { + lastUpdate: statsResult.stats.lastUpdate, + totalRules: statsResult.stats.totalRules, + databaseHealth: statsResult.stats.databaseHealth, + operatingSystems: statsResult.stats.operatingSystems, + severityLevels: statsResult.stats.severityLevels, + mitreTactics: statsResult.stats.mitreTactics, + topAuthors: statsResult.stats.topAuthors, + // Add any other statistics needed + }; + logger.info(`${FILE_NAME}: Successfully collected database statistics`); return { success: true, - stats: statsResult.stats + stats: formattedStats, + // Include raw response data for direct use by CLI + responseData: formattedStats }; } catch (error) { logger.error(`${FILE_NAME}: Error processing statistics: ${error.message}`); diff --git a/src/utils/cli_formatters.js b/src/utils/cli_formatters.js new file mode 100644 index 0000000..49636cd --- /dev/null +++ b/src/utils/cli_formatters.js @@ -0,0 +1,171 @@ +/** + * cli_formatters.js + * + * Dedicated formatters for CLI output of various data types + * Converts raw data into formatted CLI-friendly displays + */ +const chalk = require('chalk'); + +/** + * Format Sigma statistics for CLI display + * + * @param {Object} stats - The statistics object + * @returns {Object} Formatted stats ready for CLI display + */ +function formatSigmaStats(stats) { + if (!stats) { + return { error: 'No statistics data available' }; + } + + // Format date + const formatDate = (dateString) => { + if (!dateString) return 'Unknown'; + try { + const date = new Date(dateString); + return date.toLocaleString(); + } catch (error) { + return dateString; + } + }; + + // Create a simplified object suitable for table display + const formattedStats = { + 'Last Update': formatDate(stats.lastUpdate), + 'Total Rules': stats.totalRules.toLocaleString(), + 'Database Health': `${stats.databaseHealth.contentPercentage}% Complete`, + + // OS breakdown + 'Windows Rules': stats.operatingSystems.windows.toLocaleString(), + 'Linux Rules': stats.operatingSystems.linux.toLocaleString(), + 'macOS Rules': stats.operatingSystems.macos.toLocaleString(), + 'Other OS Rules': stats.operatingSystems.other.toLocaleString(), + + // Add severity levels + ...(stats.severityLevels || []).reduce((acc, level) => { + const levelName = level.level + ? level.level.charAt(0).toUpperCase() + level.level.slice(1) + : 'Unknown'; + + acc[`${levelName} Severity`] = level.count.toLocaleString(); + return acc; + }, {}) + }; + + // Add top MITRE tactics if available + if (stats.mitreTactics && stats.mitreTactics.length > 0) { + stats.mitreTactics.forEach((tactic, index) => { + if (index < 5) { // Only include top 5 for brevity + const formattedTactic = tactic.tactic + .replace(/-/g, ' ') + .split(' ') + .map(word => word.charAt(0).toUpperCase() + word.slice(1)) + .join(' '); + + formattedStats[`MITRE: ${formattedTactic}`] = tactic.count.toLocaleString(); + } + }); + } + + return formattedStats; +} + +/** + * Format Sigma search results for CLI display + * + * @param {Object} searchResults - The search results object + * @returns {Object} Formatted results ready for CLI display + */ +function formatSigmaSearchResults(searchResults) { + if (!searchResults || !searchResults.results) { + return { error: 'No search results available' }; + } + + // Return a structure with results and meta info + return { + results: searchResults.results.map(rule => ({ + id: rule.id || '', + title: rule.title || '', + author: rule.author || 'Unknown', + level: rule.level || 'medium' + })), + totalCount: searchResults.totalCount || 0 + }; +} + +/** + * Format Sigma rule details for CLI display + * + * @param {Object} ruleDetails - The rule details object + * @returns {Object} Formatted details ready for CLI display + */ +function formatSigmaDetails(ruleDetails) { + if (!ruleDetails) { + return { error: 'No rule details available' }; + } + + // Filter and format the rule details for CLI display + const formattedDetails = {}; + + // Include only the most important fields for display + const fieldsToInclude = [ + 'id', 'title', 'description', 'status', 'author', + 'level', 'falsepositives', 'references', + 'created', 'modified' + ]; + + // Add detection information if available + if (ruleDetails.detection && ruleDetails.detection.condition) { + fieldsToInclude.push('detection_condition'); + formattedDetails['detection_condition'] = ruleDetails.detection.condition; + } + + // Add logsource information if available + if (ruleDetails.logsource) { + if (ruleDetails.logsource.product) { + fieldsToInclude.push('logsource_product'); + formattedDetails['logsource_product'] = ruleDetails.logsource.product; + } + + if (ruleDetails.logsource.category) { + fieldsToInclude.push('logsource_category'); + formattedDetails['logsource_category'] = ruleDetails.logsource.category; + } + + if (ruleDetails.logsource.service) { + fieldsToInclude.push('logsource_service'); + formattedDetails['logsource_service'] = ruleDetails.logsource.service; + } + } + + // Format date fields + const dateFields = ['created', 'modified']; + + for (const [key, value] of Object.entries(ruleDetails)) { + if (fieldsToInclude.includes(key)) { + // Format dates + if (dateFields.includes(key) && value) { + try { + formattedDetails[key] = new Date(value).toLocaleString(); + } catch (e) { + formattedDetails[key] = value; + } + } + // Format arrays + else if (Array.isArray(value)) { + formattedDetails[key] = value.join(', '); + } + // Default handling + else { + formattedDetails[key] = value; + } + } + } + + return formattedDetails; +} + +module.exports = { + formatSigmaStats, + formatSigmaSearchResults, + formatSigmaDetails +}; \ No newline at end of file diff --git a/src/utils/logger.js b/src/utils/logger.js index 3225e2a..29c81a8 100644 --- a/src/utils/logger.js +++ b/src/utils/logger.js @@ -1,6 +1,7 @@ /** * logger.js - * Handles logging functionality + * + * Handles logging functionality with CLI mode support */ const fs = require('fs'); const path = require('path'); @@ -25,12 +26,29 @@ if (!fs.existsSync(LOGS_DIR)) { } // Use log file from config if available, otherwise use default -const LOG_FILE = LOGGING_CONFIG?.file - ? path.resolve(path.join(__dirname, '..', '..'), LOGGING_CONFIG.file) +const LOG_FILE = LOGGING_CONFIG?.file + ? path.resolve(path.join(__dirname, '..', '..'), LOGGING_CONFIG.file) : path.join(LOGS_DIR, 'fylgja.log'); +// Flag to determine if we're running in CLI mode +let isCliMode = false; + // Create logger object const logger = { + /** + * Set the CLI mode flag + * @param {boolean} mode - True to enable CLI mode (no console output) + */ + setCliMode: (mode) => { + isCliMode = !!mode; + }, + + /** + * Check if running in CLI mode + * @returns {boolean} CLI mode status + */ + isCliMode: () => isCliMode, + /** * Internal method to write log entry to file and console if level meets threshold * @param {string} level - Log level (DEBUG, INFO, WARN, ERROR) @@ -39,7 +57,7 @@ const logger = { _writeToFile: (level, message) => { // Check if this log level should be displayed based on configured level const levelValue = LOG_LEVELS[level] || 0; - + if (levelValue >= configuredLevelValue) { const timestamp = new Date().toISOString(); const logEntry = `${timestamp} ${level}: ${message}\n`; @@ -48,23 +66,28 @@ const logger = { try { fs.appendFileSync(LOG_FILE, logEntry); } catch (err) { - console.error(`Failed to write to log file: ${err.message}`); + // If in CLI mode, don't output to console + if (!isCliMode) { + console.error(`Failed to write to log file: ${err.message}`); + } } - // Also log to console with appropriate method - switch (level) { - case 'ERROR': - console.error(logEntry.trim()); - break; - case 'WARN': - console.warn(logEntry.trim()); - break; - case 'DEBUG': - console.debug(logEntry.trim()); - break; - case 'INFO': - default: - console.info(logEntry.trim()); + // Only log to console if not in CLI mode + if (!isCliMode) { + switch (level) { + case 'ERROR': + console.error(logEntry.trim()); + break; + case 'WARN': + console.warn(logEntry.trim()); + break; + case 'DEBUG': + console.debug(logEntry.trim()); + break; + case 'INFO': + default: + console.info(logEntry.trim()); + } } } }, -- 2.39.5 From 657a33a1895f082c311483bf016095c206e732ca Mon Sep 17 00:00:00 2001 From: Charlotte Croce Date: Sat, 19 Apr 2025 12:44:45 -0400 Subject: [PATCH 09/15] rename explain-sigma-rules to sigma-rule-details --- src/blocks/sigma/sigma_details_block.js | 4 +- .../sigma/actions/sigma_action_core.js | 10 +- .../sigma/actions/sigma_siem_actions.js | 6 +- src/handlers/sigma/sigma_details_handler.js | 62 +++++++++-- src/services/sigma/sigma_details_service.js | 4 +- src/services/sigma/sigma_stats_service.js | 4 +- src/utils/cli_formatters.js | 101 ++++++------------ 7 files changed, 96 insertions(+), 95 deletions(-) diff --git a/src/blocks/sigma/sigma_details_block.js b/src/blocks/sigma/sigma_details_block.js index d79a410..89ca272 100644 --- a/src/blocks/sigma/sigma_details_block.js +++ b/src/blocks/sigma/sigma_details_block.js @@ -15,7 +15,7 @@ const FILE_NAME = getFileName(__filename); * @param {Object} details - The rule details object containing all rule metadata * @returns {Array} Formatted Slack blocks ready for display */ -function getRuleExplanationBlocks(details) { +function getSigmaRuleDetailsBlocks(details) { logger.debug(`${FILE_NAME}: Creating rule explanation blocks for rule: ${details?.id || 'unknown'}`); if (!details) { @@ -294,5 +294,5 @@ function getRuleExplanationBlocks(details) { } module.exports = { - getRuleExplanationBlocks + getSigmaRuleDetailsBlocks }; diff --git a/src/handlers/sigma/actions/sigma_action_core.js b/src/handlers/sigma/actions/sigma_action_core.js index 7a9af17..dd79157 100644 --- a/src/handlers/sigma/actions/sigma_action_core.js +++ b/src/handlers/sigma/actions/sigma_action_core.js @@ -5,9 +5,9 @@ */ const logger = require('../../../utils/logger'); const { handleError } = require('../../../utils/error_handler'); -const { explainSigmaRule } = require('../../../services/sigma/sigma_details_service'); +const { getSigmaRuleDetails } = require('../../../services/sigma/sigma_details_service'); const { convertRuleToBackend } = require('../../../services/sigma/sigma_backend_converter'); -const { getRuleExplanationBlocks } = require('../../../blocks/sigma/sigma_details_block'); +const { getSigmaRuleDetailsBlocks } = require('../../../blocks/sigma/sigma_details_block'); const { getConversionResultBlocks } = require('../../../blocks/sigma/sigma_conversion_block'); const { SIGMA_CLI_CONFIG } = require('../../../config/appConfig'); @@ -38,8 +38,8 @@ const processRuleDetails = async (ruleId, respond, replaceOriginal = false, resp logger.info(`${FILE_NAME}: Processing details for sigma rule: ${ruleId}`); // Get Sigma rule details - logger.info(`${FILE_NAME}: Calling explainSigmaRule with ID: '${ruleId}'`); - const result = await explainSigmaRule(ruleId); + logger.info(`${FILE_NAME}: Calling getSigmaRuleDetails with ID: '${ruleId}'`); + const result = await getSigmaRuleDetails(ruleId); if (!result.success) { logger.error(`${FILE_NAME}: Rule details retrieval failed: ${result.message}`); @@ -66,7 +66,7 @@ const processRuleDetails = async (ruleId, respond, replaceOriginal = false, resp // Generate blocks let blocks; try { - blocks = getRuleExplanationBlocks(result.explanation); + blocks = getSigmaRuleDetailsBlocks(result.explanation); } catch (blockError) { await handleError(blockError, `${FILE_NAME}: Block generation`, respond, { replaceOriginal: replaceOriginal, diff --git a/src/handlers/sigma/actions/sigma_siem_actions.js b/src/handlers/sigma/actions/sigma_siem_actions.js index d446fcd..72fc9e3 100644 --- a/src/handlers/sigma/actions/sigma_siem_actions.js +++ b/src/handlers/sigma/actions/sigma_siem_actions.js @@ -5,7 +5,7 @@ */ const logger = require('../../../utils/logger'); const { handleError } = require('../../../utils/error_handler'); -const { explainSigmaRule } = require('../../../services/sigma/sigma_details_service'); +const { getSigmaRuleDetails } = require('../../../services/sigma/sigma_details_service'); const { convertRuleToBackend } = require('../../../services/sigma/sigma_backend_converter'); const { sendRuleToSiem } = require('../../../services/elastic/elastic_send_rule_to_siem_service'); const { getAllSpaces } = require('../../../services/elastic/elastic_api_service'); @@ -210,8 +210,8 @@ const registerSiemActions = (app) => { const ruleId = actionValue.replace('select_space_for_rule_', ''); // Get rule information to display in the space selection message - const explainResult = await explainSigmaRule(ruleId); - const ruleInfo = explainResult.success ? explainResult.explanation : { title: ruleId }; + const sigmaRuleDetailsResult = await getSigmaRuleDetails(ruleId); + const ruleInfo = sigmaRuleDetailsResult.success ? sigmaRuleDetailsResult.explanation : { title: ruleId }; // Generate blocks for space selection const blocks = getSpaceSelectionBlocks(ruleId, ruleInfo); diff --git a/src/handlers/sigma/sigma_details_handler.js b/src/handlers/sigma/sigma_details_handler.js index 55844b3..e3896e8 100644 --- a/src/handlers/sigma/sigma_details_handler.js +++ b/src/handlers/sigma/sigma_details_handler.js @@ -1,27 +1,34 @@ /** * sigma_details_handler.js * - * Handles Sigma rule details requests from Slack commands + * Handles Sigma rule details requests from both Slack commands and CLI * Processes requests for rule explanations */ const logger = require('../../utils/logger'); const { handleError } = require('../../utils/error_handler'); -const { explainSigmaRule } = require('../../services/sigma/sigma_details_service'); -const { processRuleDetails } = require('./actions/sigma_action_core'); -const FILE_NAME = 'sigma_details_handler.js'; +const { getSigmaRuleDetails, getSigmaRuleYaml } = require('../../services/sigma/sigma_details_service'); +const { getSigmaRuleDetailsBlocks } = require('../../blocks/sigma/sigma_details_block'); +const { formatSigmaDetails } = require('../../utils/cli_formatters'); + +const { getFileName } = require('../../utils/file_utils'); +const FILE_NAME = getFileName(__filename); + /** * Handle the sigma-details command for Sigma rules * - * @param {Object} command - The Slack command object - * @param {Function} respond - Function to send response back to Slack + * @param {Object} command - The Slack command or CLI command object + * @param {Function} respond - Function to send response back to Slack or CLI */ const handleCommand = async (command, respond) => { try { - logger.debug(`${FILE_NAME}: Processing sigma-details command: ${JSON.stringify(command.text)}`); + logger.debug(`${FILE_NAME}: Processing sigma-details command: ${command.text}`); if (!command || !command.text) { logger.warn(`${FILE_NAME}: Empty command received for sigma-details`); - await respond('Invalid command. Usage: /sigma-details [id]'); + await respond({ + text: 'Invalid command. Usage: /sigma-details [id] or "details sigma [id]"', + response_type: 'ephemeral' + }); return; } @@ -30,7 +37,10 @@ const handleCommand = async (command, respond) => { if (!ruleId) { logger.warn(`${FILE_NAME}: Missing rule ID in sigma-details command`); - await respond('Invalid command: missing rule ID. Usage: /sigma-details [id]'); + await respond({ + text: 'Invalid command: missing rule ID. Usage: /sigma-details [id] or "details sigma [id]"', + response_type: 'ephemeral' + }); return; } @@ -40,14 +50,44 @@ const handleCommand = async (command, respond) => { response_type: 'ephemeral' }); - // Use the shared processRuleDetails function from action handlers - await processRuleDetails(ruleId, respond, false, 'in_channel'); + // Get the rule explanation + const sigmaRuleDetailsResult = await getSigmaRuleDetails(ruleId); + + if (!sigmaRuleDetailsResult.success) { + logger.warn(`${FILE_NAME}: Failed to explain rule ${ruleId}: ${sigmaRuleDetailsResult.message}`); + await respond({ + text: `Error: ${sigmaRuleDetailsResult.message}`, + response_type: 'ephemeral' + }); + return; + } + + // For Slack responses, generate Block Kit blocks + let blocks; + try { + // This is for Slack - get the Block Kit UI components + blocks = getSigmaRuleDetailsBlocks(sigmaRuleDetailsResult.explanation); + } catch (blockError) { + await handleError(blockError, `${FILE_NAME}: Block generation`, respond, { + responseType: 'ephemeral', + customMessage: 'Error generating rule details view' + }); + return; + } + + // Return the response with both blocks for Slack and responseData for CLI + await respond({ + blocks: blocks, // For Slack + responseData: sigmaRuleDetailsResult.explanation, // For CLI + response_type: 'in_channel' + }); } catch (error) { await handleError(error, `${FILE_NAME}: Details command handler`, respond, { responseType: 'ephemeral' }); } }; + module.exports = { handleCommand }; \ No newline at end of file diff --git a/src/services/sigma/sigma_details_service.js b/src/services/sigma/sigma_details_service.js index e0dd364..21031a0 100644 --- a/src/services/sigma/sigma_details_service.js +++ b/src/services/sigma/sigma_details_service.js @@ -17,7 +17,7 @@ const FILE_NAME = getFileName(__filename); * @param {string} ruleId - The ID of the rule to explain * @returns {Promise} Result object with success flag and explanation or error message */ -async function explainSigmaRule(ruleId) { +async function getSigmaRuleDetails(ruleId) { if (!ruleId) { logger.warn(`${FILE_NAME}: Cannot explain rule: Missing rule ID`); return { @@ -145,6 +145,6 @@ async function getSigmaRuleYaml(ruleId) { } module.exports = { - explainSigmaRule, + getSigmaRuleDetails, getSigmaRuleYaml }; \ No newline at end of file diff --git a/src/services/sigma/sigma_stats_service.js b/src/services/sigma/sigma_stats_service.js index 20cac15..81934d1 100644 --- a/src/services/sigma/sigma_stats_service.js +++ b/src/services/sigma/sigma_stats_service.js @@ -48,7 +48,9 @@ async function getSigmaStats() { return { success: true, stats: formattedStats, - // Include raw response data for direct use by CLI + // Include raw response data for direct use by CLI. + // We have one universal function in the CLI to receive responses, + // and the CLI will then format each result differently responseData: formattedStats }; } catch (error) { diff --git a/src/utils/cli_formatters.js b/src/utils/cli_formatters.js index 49636cd..44ae36a 100644 --- a/src/utils/cli_formatters.js +++ b/src/utils/cli_formatters.js @@ -6,6 +6,36 @@ */ const chalk = require('chalk'); + +/** + * Format Sigma rule details for CLI display + * @param {Object} ruleDetails - The rule details to format + * @returns {Object} Formatted rule details for CLI display + */ +function formatSigmaDetails(ruleDetails) { + if (!ruleDetails) { + return null; + } + + // Create a flattened object for display in CLI table format + const formattedDetails = { + 'ID': ruleDetails.id || 'Unknown', + 'Title': ruleDetails.title || 'Untitled Rule', + 'Description': ruleDetails.description || 'No description provided', + 'Author': ruleDetails.author || 'Unknown author', + 'Severity': ruleDetails.severity || 'Unknown', + 'Detection': ruleDetails.detectionExplanation || 'No detection specified', + 'False Positives': Array.isArray(ruleDetails.falsePositives) ? + ruleDetails.falsePositives.join(', ') : 'None specified', + 'Tags': Array.isArray(ruleDetails.tags) ? + ruleDetails.tags.join(', ') : 'None', + 'References': Array.isArray(ruleDetails.references) ? + ruleDetails.references.join(', ') : 'None' + }; + + return formattedDetails; +} + /** * Format Sigma statistics for CLI display * @@ -92,77 +122,6 @@ function formatSigmaSearchResults(searchResults) { }; } -/** - * Format Sigma rule details for CLI display - * - * @param {Object} ruleDetails - The rule details object - * @returns {Object} Formatted details ready for CLI display - */ -function formatSigmaDetails(ruleDetails) { - if (!ruleDetails) { - return { error: 'No rule details available' }; - } - - // Filter and format the rule details for CLI display - const formattedDetails = {}; - - // Include only the most important fields for display - const fieldsToInclude = [ - 'id', 'title', 'description', 'status', 'author', - 'level', 'falsepositives', 'references', - 'created', 'modified' - ]; - - // Add detection information if available - if (ruleDetails.detection && ruleDetails.detection.condition) { - fieldsToInclude.push('detection_condition'); - formattedDetails['detection_condition'] = ruleDetails.detection.condition; - } - - // Add logsource information if available - if (ruleDetails.logsource) { - if (ruleDetails.logsource.product) { - fieldsToInclude.push('logsource_product'); - formattedDetails['logsource_product'] = ruleDetails.logsource.product; - } - - if (ruleDetails.logsource.category) { - fieldsToInclude.push('logsource_category'); - formattedDetails['logsource_category'] = ruleDetails.logsource.category; - } - - if (ruleDetails.logsource.service) { - fieldsToInclude.push('logsource_service'); - formattedDetails['logsource_service'] = ruleDetails.logsource.service; - } - } - - // Format date fields - const dateFields = ['created', 'modified']; - - for (const [key, value] of Object.entries(ruleDetails)) { - if (fieldsToInclude.includes(key)) { - // Format dates - if (dateFields.includes(key) && value) { - try { - formattedDetails[key] = new Date(value).toLocaleString(); - } catch (e) { - formattedDetails[key] = value; - } - } - // Format arrays - else if (Array.isArray(value)) { - formattedDetails[key] = value.join(', '); - } - // Default handling - else { - formattedDetails[key] = value; - } - } - } - - return formattedDetails; -} module.exports = { formatSigmaStats, -- 2.39.5 From fd394fff360dac573de1d6cf5b24e298e35b83fa Mon Sep 17 00:00:00 2001 From: Charlotte Croce Date: Sat, 19 Apr 2025 12:55:56 -0400 Subject: [PATCH 10/15] add CLI details functionality --- src/fylgja-cli.js | 10 +++++----- src/handlers/sigma/sigma_details_handler.js | 5 ++--- src/lang/command_patterns.js | 12 ++---------- 3 files changed, 9 insertions(+), 18 deletions(-) diff --git a/src/fylgja-cli.js b/src/fylgja-cli.js index 612665f..e5dde90 100644 --- a/src/fylgja-cli.js +++ b/src/fylgja-cli.js @@ -428,13 +428,13 @@ Fylgja CLI Help Basic Sigma Commands: - search sigma - Search for Sigma rules by keyword - details sigma - Get details about a specific Sigma rule -- sigma stats - Get statistics about Sigma rules database +- stats sigma - Get statistics about Sigma rules database Advanced Sigma Search Commands: -- search sigma rules where title contains "ransomware" - Search by title -- search sigma rules where tags include privilege_escalation - Search by tags -- search sigma rules where logsource.category == "process_creation" - Search by log source -- search sigma rules where modified after 2024-01-01 - Search by modification date +- search sigma where title contains "ransomware" - Search by title +- search sigma where tags include privilege_escalation - Search by tags +- search sigma where logsource.category == "process_creation" - Search by log source +- search sigma where modified after 2024-01-01 - Search by modification date - exit or quit - Exit the CLI diff --git a/src/handlers/sigma/sigma_details_handler.js b/src/handlers/sigma/sigma_details_handler.js index e3896e8..079db7d 100644 --- a/src/handlers/sigma/sigma_details_handler.js +++ b/src/handlers/sigma/sigma_details_handler.js @@ -8,7 +8,6 @@ const logger = require('../../utils/logger'); const { handleError } = require('../../utils/error_handler'); const { getSigmaRuleDetails, getSigmaRuleYaml } = require('../../services/sigma/sigma_details_service'); const { getSigmaRuleDetailsBlocks } = require('../../blocks/sigma/sigma_details_block'); -const { formatSigmaDetails } = require('../../utils/cli_formatters'); const { getFileName } = require('../../utils/file_utils'); const FILE_NAME = getFileName(__filename); @@ -77,8 +76,8 @@ const handleCommand = async (command, respond) => { // Return the response with both blocks for Slack and responseData for CLI await respond({ - blocks: blocks, // For Slack - responseData: sigmaRuleDetailsResult.explanation, // For CLI + blocks: blocks, // For Slack interface + responseData: sigmaRuleDetailsResult.explanation, // For CLI interface response_type: 'in_channel' }); } catch (error) { diff --git a/src/lang/command_patterns.js b/src/lang/command_patterns.js index bcd38f3..fccdc22 100644 --- a/src/lang/command_patterns.js +++ b/src/lang/command_patterns.js @@ -17,20 +17,12 @@ const commandPatterns = [ // Sigma details patterns { - name: 'sigma-details-direct', - regex: /^(explain|get|show|display|details|info|about)\s+(rule|detection)\s+(from\s+)?sigma\s+(where\s+)?(id=|id\s+is\s+|with\s+id\s+)(.+)$/i, - action: 'details', - module: 'sigma', - params: [6] // rule ID is in capturing group 6 - }, - { - name: 'sigma-details-simple', - regex: /^(details|explain)\s+(.+)$/i, + name: 'sigma-details', + regex: /^sigma\s+(details|info|about)\s+(.+)$/i, action: 'details', module: 'sigma', params: [2] // rule ID is in capturing group 2 }, - // Sigma search patterns { name: 'sigma-search', -- 2.39.5 From 845440962d238d460752323b27a5ee282f2b8aac Mon Sep 17 00:00:00 2001 From: Charlotte Croce Date: Sat, 19 Apr 2025 13:18:38 -0400 Subject: [PATCH 11/15] format CLI tables --- src/fylgja-cli.js | 255 +++++++++++++++++++++++++----------- src/utils/cli_formatters.js | 58 ++++++-- 2 files changed, 226 insertions(+), 87 deletions(-) diff --git a/src/fylgja-cli.js b/src/fylgja-cli.js index e5dde90..5ef1a0a 100644 --- a/src/fylgja-cli.js +++ b/src/fylgja-cli.js @@ -36,10 +36,10 @@ const { handleCommand: handleConfig } = require('./handlers/config/config_handle const { handleCommand: handleStats } = require('./handlers/stats/stats_handler'); // Import CLI formatters -const { +const { formatSigmaStats, - formatSigmaSearchResults, - formatSigmaDetails + formatSigmaSearchResults, + formatSigmaDetails } = require('./utils/cli_formatters'); // Set logger to CLI mode (prevents console output) @@ -86,24 +86,83 @@ const rl = readline.createInterface({ */ function completer(line) { const commands = [ - 'search sigma', - 'details sigma', + 'search sigma', + 'details sigma', 'sigma stats', 'stats sigma', 'search sigma rules where title contains', 'search rules where tags include', 'search rules where logsource.category ==', 'search rules where modified after', - 'help', - 'exit', + 'help', + 'exit', 'quit', 'clear' ]; - + const hits = commands.filter((c) => c.startsWith(line)); return [hits.length ? hits : commands, line]; } +/** + * Normalize and wrap text for table display + * @param {string} text Text to normalize and wrap + * @param {number} maxWidth Maximum width per line + * @returns {string[]} Array of wrapped lines + */ +function normalizeAndWrap(text, maxWidth) { + if (!text) return ['']; + + // Convert to string and normalize newlines + text = String(text || ''); + + // Replace all literal newlines with spaces + text = text.replace(/\n/g, ' '); + + // Now apply word wrapping + if (text.length <= maxWidth) return [text]; + + const words = text.split(' '); + const lines = []; + let currentLine = ''; + + for (const word of words) { + // Skip empty words (could happen if there were multiple spaces) + if (!word) continue; + + // If adding this word would exceed max width + if ((currentLine.length + word.length + (currentLine ? 1 : 0)) > maxWidth) { + // Push current line if not empty + if (currentLine) { + lines.push(currentLine); + currentLine = ''; + } + + // If the word itself is longer than maxWidth, we need to split it + if (word.length > maxWidth) { + let remaining = word; + while (remaining.length > 0) { + const chunk = remaining.substring(0, maxWidth); + lines.push(chunk); + remaining = remaining.substring(maxWidth); + } + } else { + currentLine = word; + } + } else { + // Add word to current line + currentLine = currentLine ? `${currentLine} ${word}` : word; + } + } + + // Add the last line if not empty + if (currentLine) { + lines.push(currentLine); + } + + return lines; +} + /** * Format CLI output similar to MySQL * @param {Object} data The data to format @@ -117,59 +176,105 @@ function formatOutput(data, type) { switch (type) { case 'search_results': + // Search results table format remains the same console.log('\n+-------+----------------------+------------------+-------------+'); console.log('| ID | Title | Author | Level |'); console.log('+-------+----------------------+------------------+-------------+'); - + if (data.results && data.results.length > 0) { data.results.forEach(rule => { const id = (rule.id || '').padEnd(5).substring(0, 5); const title = (rule.title || '').padEnd(20).substring(0, 20); const author = (rule.author || 'Unknown').padEnd(16).substring(0, 16); const level = (rule.level || 'medium').padEnd(11).substring(0, 11); - + console.log(`| ${id} | ${title} | ${author} | ${level} |`); }); } else { console.log('| No results found |'); } - + console.log('+-------+----------------------+------------------+-------------+'); console.log(`${data.totalCount || 0} rows in set`); break; - + case 'details': - console.log('\n+----------------------+--------------------------------------------------+'); - console.log('| Field | Value |'); - console.log('+----------------------+--------------------------------------------------+'); - + // Set a fixed width for the entire table + const sigmaDetailsKeyWidth = 22; + const sigmaDetailsValueWidth = 50; + + // Create the table borders + const detailsHeaderLine = '╔' + '═'.repeat(sigmaDetailsKeyWidth) + '╦' + '═'.repeat(sigmaDetailsValueWidth) + '╗'; + const sigmaDetailsDividerLine = '╠' + '═'.repeat(sigmaDetailsKeyWidth) + '╬' + '═'.repeat(sigmaDetailsValueWidth) + '╣'; + const sigmaDetailsRowSeparator = '╟' + '─'.repeat(sigmaDetailsKeyWidth) + '╫' + '─'.repeat(sigmaDetailsValueWidth) + '╢'; + const sigmaDetailsFooterLine = '╚' + '═'.repeat(sigmaDetailsKeyWidth) + '╩' + '═'.repeat(sigmaDetailsValueWidth) + '╝'; + + console.log('\n' + detailsHeaderLine); + console.log(`║ ${'Field'.padEnd(sigmaDetailsKeyWidth - 2)} ║ ${'Value'.padEnd(sigmaDetailsValueWidth - 2)} ║`); + console.log(sigmaDetailsDividerLine); + + // Track whether we need to add a row separator + let isFirstRow = true; + for (const [key, value] of Object.entries(data)) { if (typeof value !== 'object' || value === null) { - const formattedKey = key.padEnd(20).substring(0, 20); - const formattedValue = String(value || '').padEnd(48).substring(0, 48); - - console.log(`| ${formattedKey} | ${formattedValue} |`); + // Add separator between rows (but not before the first row) + if (!isFirstRow) { + console.log(sigmaDetailsRowSeparator); + } + isFirstRow = false; + + const formattedKey = key.padEnd(sigmaDetailsKeyWidth - 2); + + // Handle wrapping + const lines = normalizeAndWrap(value, sigmaDetailsValueWidth - 2); + + // Print first line with the key + console.log(`║ ${formattedKey} ║ ${lines[0].padEnd(sigmaDetailsValueWidth - 2)} ║`); + + // Print additional lines if there are any + for (let i = 1; i < lines.length; i++) { + console.log(`║ ${' '.repeat(sigmaDetailsKeyWidth - 2)} ║ ${lines[i].padEnd(sigmaDetailsValueWidth - 2)} ║`); + } } } - - console.log('+----------------------+--------------------------------------------------+'); + + console.log(sigmaDetailsFooterLine); break; - case 'stats': - console.log('\n+--------------------+---------------+'); - console.log('| Metric | Value |'); - console.log('+--------------------+---------------+'); - + // Set column widths + const sigmaStatsMetricWidth = 25; + const sigmaStatsValueWidth = 26; + + // Create the table borders + const sigmaStatsHeaderLine = '╔' + '═'.repeat(sigmaStatsMetricWidth) + '╦' + '═'.repeat(sigmaStatsValueWidth) + '╗'; + const sigmaStatsDividerLine = '╠' + '═'.repeat(sigmaStatsMetricWidth) + '╬' + '═'.repeat(sigmaStatsValueWidth) + '╣'; + const sigmaStatsRowSeparator = '╟' + '─'.repeat(sigmaStatsMetricWidth) + '╫' + '─'.repeat(sigmaStatsValueWidth) + '╢'; + const sigmaStatsFooterLine = '╚' + '═'.repeat(sigmaStatsMetricWidth) + '╩' + '═'.repeat(sigmaStatsValueWidth) + '╝'; + + console.log('\n' + sigmaStatsHeaderLine); + console.log(`║ ${'Metric'.padEnd(sigmaStatsMetricWidth - 2)} ║ ${'Value'.padEnd(sigmaStatsValueWidth - 2)} ║`); + console.log(sigmaStatsDividerLine); + + // Track whether we need to add a row separator + let statsIsFirstRow = true; + for (const [key, value] of Object.entries(data)) { - const formattedKey = key.padEnd(18).substring(0, 18); - const formattedValue = String(value || '').padEnd(13).substring(0, 13); - - console.log(`| ${formattedKey} | ${formattedValue} |`); + // Add separator between rows (but not before the first row) + if (!statsIsFirstRow) { + console.log(sigmaStatsRowSeparator); + } + statsIsFirstRow = false; + + const formattedKey = key.padEnd(sigmaStatsMetricWidth - 2); + const formattedValue = String(value || '').padEnd(sigmaStatsValueWidth - 2); + + console.log(`║ ${formattedKey} ║ ${formattedValue} ║`); } - - console.log('+--------------------+---------------+'); + + console.log(sigmaStatsFooterLine); break; - + default: console.log(JSON.stringify(data, null, 2)); } @@ -183,18 +288,18 @@ function formatOutput(data, type) { */ function extractSearchKeywords(input) { if (!input) return ''; - + // Try to extract keywords from common patterns if (input.includes('title contains')) { const match = input.match(/title\s+contains\s+["']([^"']+)["']/i); if (match) return match[1]; } - + if (input.includes('tags include')) { const match = input.match(/tags\s+include\s+(\S+)/i); if (match) return match[1]; } - + // Default - just return the input as is return input; } @@ -210,28 +315,28 @@ async function processCommand(input) { rl.prompt(); return; } - + // Special CLI commands if (input.trim().toLowerCase() === 'exit' || input.trim().toLowerCase() === 'quit') { console.log('Goodbye!'); rl.close(); process.exit(0); } - + if (input.trim().toLowerCase() === 'clear') { console.clear(); rl.prompt(); return; } - + // Special case for simple search if (input.trim().match(/^search\s+sigma\s+(.+)$/i)) { const keyword = input.trim().match(/^search\s+sigma\s+(.+)$/i)[1]; - + // Add to command history commandHistory.push(input); historyIndex = commandHistory.length; - + // Create fake command object const command = { text: keyword, @@ -241,12 +346,12 @@ async function processCommand(input) { channel_id: 'cli', channel_name: 'cli' }; - + // Create custom respond function const respond = createRespondFunction('search', 'sigma', [keyword]); - + console.log(`Executing: module=sigma, action=search, params=[${keyword}]`); - + try { await sigmaSearchHandler.handleCommand(command, respond); } catch (error) { @@ -255,29 +360,29 @@ async function processCommand(input) { logger.debug(`${FILE_NAME}: Error stack: ${error.stack}`); rl.prompt(); } - + return; } - + // Add to command history commandHistory.push(input); historyIndex = commandHistory.length; - + // Parse command using existing parser const parsedCommand = await parseCommand(input); - + if (!parsedCommand.success) { console.log(parsedCommand.message || "Command not recognized. Type 'help' for usage."); rl.prompt(); return; } - + // Extract the command details const { action, module, params } = parsedCommand.command; - + // Only show execution info to the user, not sending to logger console.log(`Executing: module=${module}, action=${action}, params=[${params}]`); - + // Create fake command object similar to Slack's const command = { text: Array.isArray(params) && params.length > 0 ? params[0] : input, @@ -287,17 +392,17 @@ async function processCommand(input) { channel_id: 'cli', channel_name: 'cli' }; - + // Special handling for complexSearch to extract keywords if (action === 'complexSearch' && module === 'sigma' && params.length > 0) { // Try to extract keywords from complex queries const searchTerms = extractSearchKeywords(params[0]); command.text = searchTerms || params[0]; } - + // Create custom respond function for CLI const respond = createRespondFunction(action, module, params); - + try { switch (module) { case 'sigma': @@ -305,50 +410,50 @@ async function processCommand(input) { case 'search': await sigmaSearchHandler.handleCommand(command, respond); break; - + case 'complexSearch': await sigmaSearchHandler.handleComplexSearch(command, respond); break; - + case 'details': await sigmaDetailsHandler.handleCommand(command, respond); break; - + case 'stats': await sigmaStatsHandler.handleCommand(command, respond); break; - + case 'create': await sigmaCreateHandler.handleCommand(command, respond); break; - + default: console.log(`Unknown Sigma action: ${action}`); rl.prompt(); } break; - + case 'alerts': await handleAlerts(command, respond); break; - + case 'case': await handleCase(command, respond); break; - + case 'config': await handleConfig(command, respond); break; - + case 'stats': await handleStats(command, respond); break; - + case 'help': displayHelp(); rl.prompt(); break; - + default: console.log(`Unknown module: ${module}`); rl.prompt(); @@ -383,13 +488,13 @@ function createRespondFunction(action, module, params) { rl.prompt(); return; } - + // First check for the responseData property (directly from service) if (response.responseData) { // Format the data using the appropriate formatter if (module === 'sigma') { let formattedData; - + if (action === 'search' || action === 'complexSearch') { formattedData = formatSigmaSearchResults(response.responseData); formatOutput(formattedData, 'search_results'); @@ -413,7 +518,7 @@ function createRespondFunction(action, module, params) { } else { console.log('Command completed successfully.'); } - + rl.prompt(); }; } @@ -441,7 +546,7 @@ Advanced Sigma Search Commands: - clear - Clear the terminal screen - help - Display this help text `; - + console.log(helpText); } @@ -452,7 +557,7 @@ function startCLI() { console.log(ASCII_LOGO); console.log(`Fylgja CLI v${version} - Interactive SIEM Management Tool`); console.log(`Type 'help' for usage information or 'exit' to quit\n`); - + // Set up key bindings for history navigation rl._writeToOutput = function _writeToOutput(stringToWrite) { if (stringToWrite === '\\u001b[A' || stringToWrite === '\\u001b[B') { @@ -461,7 +566,7 @@ function startCLI() { } rl.output.write(stringToWrite); }; - + // Set up key listeners for history rl.input.on('keypress', (char, key) => { if (key && key.name === 'up') { @@ -485,13 +590,13 @@ function startCLI() { } } }); - + rl.prompt(); - + rl.on('line', async (line) => { await processCommand(line.trim()); }); - + rl.on('close', () => { console.log('Goodbye!'); process.exit(0); diff --git a/src/utils/cli_formatters.js b/src/utils/cli_formatters.js index 44ae36a..9914284 100644 --- a/src/utils/cli_formatters.js +++ b/src/utils/cli_formatters.js @@ -6,6 +6,40 @@ */ const chalk = require('chalk'); +/** + * Wraps text at specified length + * @param {string} text - Text to wrap + * @param {number} maxLength - Maximum line length + * @returns {string} Wrapped text + */ +function wrapText(text, maxLength = 80) { + if (!text || typeof text !== 'string') { + return text; + } + + if (text.length <= maxLength) { + return text; + } + + const words = text.split(' '); + let wrappedText = ''; + let currentLine = ''; + + words.forEach(word => { + // If adding this word would exceed max length, start a new line + if ((currentLine + word).length + 1 > maxLength) { + wrappedText += currentLine.trim() + '\n'; + currentLine = word + ' '; + } else { + currentLine += word + ' '; + } + }); + + // Add the last line + wrappedText += currentLine.trim(); + + return wrappedText; +} /** * Format Sigma rule details for CLI display @@ -20,17 +54,17 @@ function formatSigmaDetails(ruleDetails) { // Create a flattened object for display in CLI table format const formattedDetails = { 'ID': ruleDetails.id || 'Unknown', - 'Title': ruleDetails.title || 'Untitled Rule', - 'Description': ruleDetails.description || 'No description provided', + 'Title': wrapText(ruleDetails.title || 'Untitled Rule', 80), + 'Description': wrapText(ruleDetails.description || 'No description provided', 80), 'Author': ruleDetails.author || 'Unknown author', 'Severity': ruleDetails.severity || 'Unknown', - 'Detection': ruleDetails.detectionExplanation || 'No detection specified', - 'False Positives': Array.isArray(ruleDetails.falsePositives) ? - ruleDetails.falsePositives.join(', ') : 'None specified', - 'Tags': Array.isArray(ruleDetails.tags) ? - ruleDetails.tags.join(', ') : 'None', - 'References': Array.isArray(ruleDetails.references) ? - ruleDetails.references.join(', ') : 'None' + 'Detection': wrapText(ruleDetails.detectionExplanation || 'No detection specified', 80), + 'False Positives': wrapText(Array.isArray(ruleDetails.falsePositives) ? + ruleDetails.falsePositives.join(', ') : 'None specified', 80), + 'Tags': wrapText(Array.isArray(ruleDetails.tags) ? + ruleDetails.tags.join(', ') : 'None', 80), + 'References': wrapText(Array.isArray(ruleDetails.references) ? + ruleDetails.references.join(', ') : 'None', 80) }; return formattedDetails; @@ -114,7 +148,7 @@ function formatSigmaSearchResults(searchResults) { return { results: searchResults.results.map(rule => ({ id: rule.id || '', - title: rule.title || '', + title: wrapText(rule.title || '', 60), // Use narrower width for table columns author: rule.author || 'Unknown', level: rule.level || 'medium' })), @@ -122,9 +156,9 @@ function formatSigmaSearchResults(searchResults) { }; } - module.exports = { formatSigmaStats, formatSigmaSearchResults, - formatSigmaDetails + formatSigmaDetails, + wrapText }; \ No newline at end of file -- 2.39.5 From 2d49dbdc46b1ce54616f59d398b2596a519a5ef3 Mon Sep 17 00:00:00 2001 From: Charlotte Croce Date: Sat, 19 Apr 2025 13:24:52 -0400 Subject: [PATCH 12/15] update details sigma command pattern --- src/lang/command_patterns.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/lang/command_patterns.js b/src/lang/command_patterns.js index fccdc22..b95c582 100644 --- a/src/lang/command_patterns.js +++ b/src/lang/command_patterns.js @@ -18,10 +18,10 @@ const commandPatterns = [ // Sigma details patterns { name: 'sigma-details', - regex: /^sigma\s+(details|info|about)\s+(.+)$/i, + regex: /^details\s+sigma\s+(.+)$/i, action: 'details', module: 'sigma', - params: [2] // rule ID is in capturing group 2 + params: [1] // rule ID is in capturing group 1 }, // Sigma search patterns { -- 2.39.5 From ad6b108d3feb6b92eec8d6f4a091c165118fcd19 Mon Sep 17 00:00:00 2001 From: Charlotte Croce Date: Sat, 19 Apr 2025 14:02:30 -0400 Subject: [PATCH 13/15] add os and category labels to details and conversion slack blocks --- src/blocks/sigma/sigma_conversion_block.js | 20 +++++++++ src/blocks/sigma/sigma_details_block.js | 24 +++++++++- .../sigma/sigma_search_results_block.js | 17 ++++++- src/handlers/sigma/sigma_search_handler.js | 5 ++- src/services/sigma/sigma_details_service.js | 45 ++++++++++--------- 5 files changed, 84 insertions(+), 27 deletions(-) diff --git a/src/blocks/sigma/sigma_conversion_block.js b/src/blocks/sigma/sigma_conversion_block.js index 8190e6f..3932a9f 100644 --- a/src/blocks/sigma/sigma_conversion_block.js +++ b/src/blocks/sigma/sigma_conversion_block.js @@ -40,6 +40,13 @@ function getConversionResultBlocks(conversionResult) { format: 'siem_rule_ndjson' }; + // Extract logsource information or use defaults + const logsource = rule.logsource || {}; + const product = logsource.product || 'N/A'; + const category = logsource.category || 'N/A'; + + logger.debug(`${FILE_NAME}: Logsource info - Product: ${product}, Category: ${category}`); + // Truncate output if it's too long for Slack let output = conversionResult.output || ''; const maxOutputLength = 2900; // Slack has a limit of ~3000 chars in a code block @@ -66,6 +73,19 @@ function getConversionResultBlocks(conversionResult) { text: `*Rule ID:* ${rule.id}\n*Description:* ${rule.description}` } }, + { + type: 'section', + fields: [ + { + type: 'mrkdwn', + text: `*OS/Product:* ${product}` + }, + { + type: 'mrkdwn', + text: `*Category:* ${category}` + } + ] + }, { type: 'section', text: { diff --git a/src/blocks/sigma/sigma_details_block.js b/src/blocks/sigma/sigma_details_block.js index 89ca272..ccf175b 100644 --- a/src/blocks/sigma/sigma_details_block.js +++ b/src/blocks/sigma/sigma_details_block.js @@ -193,6 +193,28 @@ function getSigmaRuleDetailsBlocks(details) { } ]; + // Get logsource information from the details + const logsource = details.logsource || {}; + const product = logsource.product || 'N/A'; + const category = logsource.category || 'N/A'; + + logger.debug(`${FILE_NAME}: Logsource info - Product: ${product}, Category: ${category}`); + + // Add logsource information section after severity/author + blocks.push({ + type: 'section', + fields: [ + { + type: 'mrkdwn', + text: `*OS/Product:* ${product}` + }, + { + type: 'mrkdwn', + text: `*Category:* ${category}` + } + ] + }); + // Add divider for visual separation blocks.push({ type: 'divider' }); @@ -295,4 +317,4 @@ function getSigmaRuleDetailsBlocks(details) { module.exports = { getSigmaRuleDetailsBlocks -}; +}; \ No newline at end of file diff --git a/src/blocks/sigma/sigma_search_results_block.js b/src/blocks/sigma/sigma_search_results_block.js index e118c17..6ab7aef 100644 --- a/src/blocks/sigma/sigma_search_results_block.js +++ b/src/blocks/sigma/sigma_search_results_block.js @@ -78,12 +78,25 @@ const getSearchResultBlocks = (keyword, results, pagination = {}) => { const ruleId = safeRule.id || 'unknown'; logger.debug(`${FILE_NAME}: Adding result #${index + 1}: ${ruleId} - ${safeRule.title || 'Untitled'}`); - // Combine rule information and action button into a single line + // Get OS emoji based on product + const getOsEmoji = (product) => { + if (!product) return ''; + + const productLower = product.toLowerCase(); + if (productLower.includes('windows')) return ':window: '; + if (productLower.includes('mac') || productLower.includes('apple')) return ':apple: '; + if (productLower.includes('linux')) return ':penguin: '; + return ''; + }; + + const osEmoji = getOsEmoji(safeRule.logsource && safeRule.logsource.product); + + // Rule information and action button - with OS emoji before title and no ID field blocks.push({ "type": "section", "text": { "type": "mrkdwn", - "text": `*${safeRule.title || 'Untitled Rule'}*\nID: \`${ruleId}\`` + "text": `*${osEmoji}${safeRule.title || 'Untitled Rule'}*` }, "accessory": { "type": "button", diff --git a/src/handlers/sigma/sigma_search_handler.js b/src/handlers/sigma/sigma_search_handler.js index f6d9e17..2932b8d 100644 --- a/src/handlers/sigma/sigma_search_handler.js +++ b/src/handlers/sigma/sigma_search_handler.js @@ -4,7 +4,7 @@ * Handles Sigma rule search requests from Slack commands */ -const { searchSigmaRules, searchSigmaRulesComplex } = require('../../services/sigma/sigma_search_service'); +const { searchSigmaRules, searchSigmaRulesComplex, searchAndConvertRules } = require('../../services/sigma/sigma_search_service'); const logger = require('../../utils/logger'); const { handleError } = require('../../utils/error_handler'); const { getSearchResultBlocks } = require('../../blocks/sigma/sigma_search_results_block'); @@ -69,7 +69,7 @@ const handleCommand = async (command, respond) => { }); // Search for rules using the service function with pagination - const searchResult = await searchSigmaRules(keyword, page, pageSize); + const searchResult = await searchAndConvertRules(keyword, page, pageSize); logger.debug(`${FILE_NAME}: Search result status: ${searchResult.success}`); logger.debug(`${FILE_NAME}: Found ${searchResult.results?.length || 0} results out of ${searchResult.pagination?.totalResults || 0} total matches`); logger.debug(`${FILE_NAME}: About to generate blocks for search results`); @@ -240,6 +240,7 @@ const handleComplexSearch = async (command, respond) => { ); // Replace the header to indicate it's a complex search + // TODO: should be moved to dedicated block file if (blocks && blocks.length > 0) { blocks[0] = { type: "header", diff --git a/src/services/sigma/sigma_details_service.js b/src/services/sigma/sigma_details_service.js index 21031a0..021611b 100644 --- a/src/services/sigma/sigma_details_service.js +++ b/src/services/sigma/sigma_details_service.js @@ -25,15 +25,15 @@ async function getSigmaRuleDetails(ruleId) { message: 'Missing rule ID' }; } - + logger.info(`${FILE_NAME}: Running diagnostics for rule: ${ruleId}`); logger.info(`${FILE_NAME}: Explaining rule ${ruleId}`); - + try { // Run diagnostics on the rule content first const diagnosticResult = await debugRuleContent(ruleId); logger.debug(`${FILE_NAME}: Diagnostic result: ${JSON.stringify(diagnosticResult || {})}`); - + // Convert the rule ID to a structured object const conversionResult = await convertSigmaRule(ruleId); if (!conversionResult.success) { @@ -43,9 +43,9 @@ async function getSigmaRuleDetails(ruleId) { message: conversionResult.message || `Failed to parse rule with ID ${ruleId}` }; } - + const rule = conversionResult.rule; - + // Extra safety check if (!rule) { logger.error(`${FILE_NAME}: Converted rule is null for ID ${ruleId}`); @@ -54,7 +54,7 @@ async function getSigmaRuleDetails(ruleId) { message: `Failed to process rule with ID ${ruleId}` }; } - + // Create a simplified explanation with safe access to properties const explanation = { id: rule.id || ruleId, @@ -62,27 +62,28 @@ async function getSigmaRuleDetails(ruleId) { description: rule.description || 'No description provided', author: rule.author || 'Unknown author', severity: rule.level || 'Unknown', + logsource: rule.logsource || {}, // Add this line to include logsource info detectionExplanation: extractDetectionCondition(rule), falsePositives: Array.isArray(rule.falsepositives) ? rule.falsepositives : - typeof rule.falsepositives === 'string' ? [rule.falsepositives] : - ['None specified'], + typeof rule.falsepositives === 'string' ? [rule.falsepositives] : + ['None specified'], tags: Array.isArray(rule.tags) ? rule.tags : [], references: Array.isArray(rule.references) ? rule.references : [] }; - + logger.info(`${FILE_NAME}: Successfully explained rule ${ruleId}`); logger.debug(`${FILE_NAME}: Explanation properties: ${Object.keys(explanation).join(', ')}`); - - return { - success: true, - explanation + + return { + success: true, + explanation }; } catch (error) { logger.error(`${FILE_NAME}: Error explaining rule: ${error.message}`); logger.debug(`${FILE_NAME}: Error stack: ${error.stack}`); - return { - success: false, - message: `Error explaining rule: ${error.message}` + return { + success: false, + message: `Error explaining rule: ${error.message}` }; } } @@ -102,13 +103,13 @@ async function getSigmaRuleYaml(ruleId) { message: 'Missing rule ID' }; } - + logger.info(`${FILE_NAME}: Getting YAML content for rule: ${ruleId}`); - + try { // Get YAML content from database const yamlResult = await getRuleYamlContent(ruleId); - + if (!yamlResult.success) { logger.warn(`${FILE_NAME}: Failed to retrieve YAML for rule ${ruleId}: ${yamlResult.message}`); return { @@ -116,7 +117,7 @@ async function getSigmaRuleYaml(ruleId) { message: yamlResult.message || `Failed to retrieve YAML for rule with ID ${ruleId}` }; } - + // Add extra safety check for content if (!yamlResult.content) { logger.warn(`${FILE_NAME}: YAML content is empty for rule ${ruleId}`); @@ -126,9 +127,9 @@ async function getSigmaRuleYaml(ruleId) { warning: 'YAML content is empty for this rule' }; } - + logger.debug(`${FILE_NAME}: Successfully retrieved YAML content with length: ${yamlResult.content.length}`); - + // Return the YAML content return { success: true, -- 2.39.5 From 977dd7e6d3848d971855b819a5d9560d2d43f565 Mon Sep 17 00:00:00 2001 From: Charlotte Croce Date: Sat, 19 Apr 2025 14:15:12 -0400 Subject: [PATCH 14/15] fixed emoji pagation issue --- .gitignore | 1 + .../sigma/sigma_search_results_block.js | 15 +---- .../sigma/actions/sigma_view_actions.js | 4 +- src/utils/os_emojis.js | 65 +++++++++++++++++++ 4 files changed, 71 insertions(+), 14 deletions(-) create mode 100644 src/utils/os_emojis.js diff --git a/.gitignore b/.gitignore index 19ad939..4608f5e 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,4 @@ fylgja.yml slack.yml sigma.db sigma-repo/ +.VSCodeCounter \ No newline at end of file diff --git a/src/blocks/sigma/sigma_search_results_block.js b/src/blocks/sigma/sigma_search_results_block.js index 6ab7aef..c459375 100644 --- a/src/blocks/sigma/sigma_search_results_block.js +++ b/src/blocks/sigma/sigma_search_results_block.js @@ -8,6 +8,7 @@ const logger = require('../../utils/logger'); const { getFileName } = require('../../utils/file_utils'); +const { getProductEmoji } = require('../../utils/os_emojis'); const FILE_NAME = getFileName(__filename); /** @@ -78,18 +79,8 @@ const getSearchResultBlocks = (keyword, results, pagination = {}) => { const ruleId = safeRule.id || 'unknown'; logger.debug(`${FILE_NAME}: Adding result #${index + 1}: ${ruleId} - ${safeRule.title || 'Untitled'}`); - // Get OS emoji based on product - const getOsEmoji = (product) => { - if (!product) return ''; - - const productLower = product.toLowerCase(); - if (productLower.includes('windows')) return ':window: '; - if (productLower.includes('mac') || productLower.includes('apple')) return ':apple: '; - if (productLower.includes('linux')) return ':penguin: '; - return ''; - }; - - const osEmoji = getOsEmoji(safeRule.logsource && safeRule.logsource.product); + // Get product emoji + const osEmoji = getProductEmoji(safeRule.logsource && safeRule.logsource.product); // Rule information and action button - with OS emoji before title and no ID field blocks.push({ diff --git a/src/handlers/sigma/actions/sigma_view_actions.js b/src/handlers/sigma/actions/sigma_view_actions.js index d896f31..09d07f3 100644 --- a/src/handlers/sigma/actions/sigma_view_actions.js +++ b/src/handlers/sigma/actions/sigma_view_actions.js @@ -6,7 +6,7 @@ const logger = require('../../../utils/logger'); const { handleError } = require('../../../utils/error_handler'); const { getSigmaRuleYaml } = require('../../../services/sigma/sigma_details_service'); -const { searchSigmaRules } = require('../../../services/sigma/sigma_search_service'); +const { searchSigmaRules, searchAndConvertRules } = require('../../../services/sigma/sigma_search_service'); const { getYamlViewBlocks } = require('../../../blocks/sigma/sigma_view_yaml_block'); const { getSearchResultBlocks } = require('../../../blocks/sigma/sigma_search_results_block'); const { processRuleDetails } = require('./sigma_action_core'); @@ -62,7 +62,7 @@ const handlePaginationAction = async (body, ack, respond) => { logger.info(`${FILE_NAME}: Processing pagination request for "${keyword}" (page ${page}, size ${pageSize})`); // Perform the search with the new pagination parameters - const searchResult = await searchSigmaRules(keyword, page, pageSize); + const searchResult = await searchAndConvertRules(keyword, page, pageSize); if (!searchResult.success) { logger.error(`${FILE_NAME}: Search failed during pagination: ${searchResult.message}`); diff --git a/src/utils/os_emojis.js b/src/utils/os_emojis.js new file mode 100644 index 0000000..4d135e9 --- /dev/null +++ b/src/utils/os_emojis.js @@ -0,0 +1,65 @@ +/** + * os_emojis.js + * + * Provides emoji mappings for different products/platforms in Sigma rules + */ + +/** + * Get the appropriate emoji for a product + * @param {string} product - The product/platform name + * @returns {string} - The corresponding emoji string + */ +const getProductEmoji = (product) => { + if (!product) return ''; + + const productLower = product.toLowerCase(); + + // Mapping of products to their respective emojis + const emojiMap = { + 'aws': ':cloud:', + 'azure': ':cloud:', + 'bitbucket': ':bucket:', + 'cisco': ':satellite_antenna:', + 'django': ':snake:', + 'dns': ':globe_with_meridians:', + 'fortios': ':shield:', + 'gcp': ':cloud:', + 'github': ':octocat:', + 'huawei': ':satellite_antenna:', + 'juniper': ':satellite_antenna:', + 'jvm': ':coffee:', + 'kubernetes': ':wheel_of_dharma:', + 'linux': ':penguin:', + 'm365': ':envelope:', + 'macos': ':apple:', + 'modsecurity': ':shield:', + 'nodejs': ':green_heart:', + 'okta': ':key:', + 'onelogin': ':key:', + 'opencanary': ':bird:', + 'paloalto': ':shield:', + 'python': ':snake:', + 'qualys': ':mag:', + 'rpc_firewall': ':fire_extinguisher:', + 'ruby_on_rails': ':gem:', + 'spring': ':leaves:', + 'sql': ':floppy_disk:', + 'velocity': ':zap:', + 'windows': ':window:', + 'zeek': ':eyes:' + }; + + // Check if the product is directly in our map + for (const [key, emoji] of Object.entries(emojiMap)) { + if (productLower.includes(key)) { + return emoji + ' '; + } + } + + // Default emoji for unknown products + return ':computer: '; + }; + + module.exports = { + getProductEmoji + }; \ No newline at end of file -- 2.39.5 From 96892cb94b629b3e6773b1ebc49a69c0dd16b06b Mon Sep 17 00:00:00 2001 From: Charlotte Croce Date: Sat, 19 Apr 2025 14:18:22 -0400 Subject: [PATCH 15/15] fix command-and-control tag link --- src/blocks/sigma/sigma_details_block.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/blocks/sigma/sigma_details_block.js b/src/blocks/sigma/sigma_details_block.js index ccf175b..5ac5217 100644 --- a/src/blocks/sigma/sigma_details_block.js +++ b/src/blocks/sigma/sigma_details_block.js @@ -112,7 +112,7 @@ function getSigmaRuleDetailsBlocks(details) { 'discovery': 'TA0007', 'lateralmovement': 'TA0008', 'collection': 'TA0009', - 'command-and-control': 'TA0011', + 'commandandcontrol': 'TA0011', 'exfiltration': 'TA0010', 'impact': 'TA0040' }; -- 2.39.5