From 85bb8958b8139163721f584de44abe8f5f6e2bf1 Mon Sep 17 00:00:00 2001 From: Charlotte Croce Date: Fri, 18 Apr 2025 15:45:26 -0400 Subject: [PATCH] refactor sigma_db_queries into multiple files --- .vscode/settings.json | 2 + src/services/sigma/sigma_backend_converter.js | 2 +- src/services/sigma/sigma_converter_service.js | 2 +- src/services/sigma/sigma_details_service.js | 4 +- src/services/sigma/sigma_search_service.js | 2 +- src/services/sigma/sigma_stats_service.js | 2 +- src/sigma_db/queries/complex-search.js | 259 ++++ src/sigma_db/queries/fts-search.js | 133 ++ src/sigma_db/queries/index.js | 34 + src/sigma_db/queries/query-builders.js | 258 ++++ src/sigma_db/queries/rule-retrieval.js | 164 +++ src/sigma_db/queries/simple-search.js | 118 ++ src/sigma_db/queries/stats-debug.js | 328 +++++ src/sigma_db/sigma_db_queries.js | 1207 ----------------- 14 files changed, 1302 insertions(+), 1213 deletions(-) create mode 100644 .vscode/settings.json create mode 100644 src/sigma_db/queries/complex-search.js create mode 100644 src/sigma_db/queries/fts-search.js create mode 100644 src/sigma_db/queries/index.js create mode 100644 src/sigma_db/queries/query-builders.js create mode 100644 src/sigma_db/queries/rule-retrieval.js create mode 100644 src/sigma_db/queries/simple-search.js create mode 100644 src/sigma_db/queries/stats-debug.js delete mode 100644 src/sigma_db/sigma_db_queries.js diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..7a73a41 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,2 @@ +{ +} \ No newline at end of file diff --git a/src/services/sigma/sigma_backend_converter.js b/src/services/sigma/sigma_backend_converter.js index b0330c4..59f63e5 100644 --- a/src/services/sigma/sigma_backend_converter.js +++ b/src/services/sigma/sigma_backend_converter.js @@ -11,7 +11,7 @@ const { execSync } = require('child_process'); const logger = require('../../utils/logger'); const { SIGMA_CLI_PATH, SIGMA_CLI_CONFIG } = require('../../config/appConfig'); const { convertSigmaRule } = require('./sigma_converter_service'); -const { getRuleYamlContent } = require('../../sigma_db/sigma_db_queries'); +const { getRuleYamlContent } = require('../../sigma_db/queries'); const { getFileName } = require('../../utils/file_utils'); const FILE_NAME = getFileName(__filename); diff --git a/src/services/sigma/sigma_converter_service.js b/src/services/sigma/sigma_converter_service.js index 5ae5679..1f57a17 100644 --- a/src/services/sigma/sigma_converter_service.js +++ b/src/services/sigma/sigma_converter_service.js @@ -4,7 +4,7 @@ // const logger = require('../../utils/logger'); const yaml = require('js-yaml'); -const { findRuleById } = require('../../sigma_db/sigma_db_queries'); +const { findRuleById } = require('../../sigma_db/queries'); const { getFileName } = require('../../utils/file_utils'); const FILE_NAME = getFileName(__filename); diff --git a/src/services/sigma/sigma_details_service.js b/src/services/sigma/sigma_details_service.js index 2f43019..e0dd364 100644 --- a/src/services/sigma/sigma_details_service.js +++ b/src/services/sigma/sigma_details_service.js @@ -5,7 +5,7 @@ */ const logger = require('../../utils/logger'); const { convertSigmaRule, extractDetectionCondition } = require('./sigma_converter_service'); -const { debugRuleContent, getRuleYamlContent } = require('../../sigma_db/sigma_db_queries'); +const { debugRuleContent, getRuleYamlContent } = require('../../sigma_db/queries'); const { getFileName } = require('../../utils/file_utils'); const FILE_NAME = getFileName(__filename); @@ -147,4 +147,4 @@ async function getSigmaRuleYaml(ruleId) { module.exports = { explainSigmaRule, getSigmaRuleYaml -}; +}; \ No newline at end of file diff --git a/src/services/sigma/sigma_search_service.js b/src/services/sigma/sigma_search_service.js index 4d17ad3..87b44f2 100644 --- a/src/services/sigma/sigma_search_service.js +++ b/src/services/sigma/sigma_search_service.js @@ -6,7 +6,7 @@ * Supports pagination for large result sets. */ -const { searchRules, searchRulesComplex } = require('../../sigma_db/sigma_db_queries'); +const { searchRules, searchRulesComplex } = require('../../sigma_db/queries'); const { parseComplexQuery } = require('../../lang/query_parser'); const logger = require('../../utils/logger'); const { convertSigmaRule } = require('./sigma_converter_service'); diff --git a/src/services/sigma/sigma_stats_service.js b/src/services/sigma/sigma_stats_service.js index c6ac0d6..0bee7a0 100644 --- a/src/services/sigma/sigma_stats_service.js +++ b/src/services/sigma/sigma_stats_service.js @@ -5,7 +5,7 @@ * Provides aggregated statistical information about the rule database */ const logger = require('../../utils/logger'); -const { getStatsFromDatabase } = require('../../sigma_db/sigma_db_queries'); +const { getStatsFromDatabase } = require('../../sigma_db/queries'); const { getFileName } = require('../../utils/file_utils'); const FILE_NAME = getFileName(__filename); diff --git a/src/sigma_db/queries/complex-search.js b/src/sigma_db/queries/complex-search.js new file mode 100644 index 0000000..8767e98 --- /dev/null +++ b/src/sigma_db/queries/complex-search.js @@ -0,0 +1,259 @@ +/** + * complex-search.js + * Functions for complex searching of Sigma rules + */ + +const { getDbConnection } = require('../sigma_db_connection'); +const logger = require('../../utils/logger'); +const { getFileName } = require('../../utils/file_utils'); +const { checkFtsAvailable } = require('./fts-search'); +const { buildComplexSqlQuery, buildComplexFtsQuery } = require('./query-builders'); +const FILE_NAME = getFileName(__filename); + +/** + * Search for Sigma rules using complex query conditions + * Supports filtering by multiple attributes like title, logsource, tags, etc. + * + * @param {Object} parsedQuery - The parsed query object containing conditions and operator + * @param {number} limit - Maximum number of results to return + * @param {number} offset - Number of results to skip (for pagination) + * @returns {Promise} Object with results array and total count + */ +async function searchRulesComplex(parsedQuery, limit = 10, offset = 0) { + if (!parsedQuery || !parsedQuery.valid) { + logger.warn(`${FILE_NAME}: Invalid query object provided`); + return { results: [], totalCount: 0 }; + } + + logger.info(`${FILE_NAME}: Performing complex search with ${parsedQuery.conditions.length} conditions (limit: ${limit}, offset: ${offset})`); + + let db; + // Declare this at function scope so it's available in the finally block + let usingFts = false; + + try { + db = await getDbConnection(); + logger.debug(`${FILE_NAME}: Database connection established for complex search`); + + // Check if FTS5 is available + const ftsAvailable = await checkFtsAvailable(db); + + if (ftsAvailable) { + logger.debug(`${FILE_NAME}: Using FTS5 for complex search`); + // Set flag that we're using FTS + usingFts = true; + // Pass db connection to searchRulesComplexFTS and let that function manage it + const results = await searchRulesComplexFTS(parsedQuery, limit, offset, db); + return results; + } + + logger.debug(`${FILE_NAME}: FTS5 not available, using legacy complex search method`); + + // Build the SQL query based on the conditions + const { sqlQuery, sqlCountQuery, params } = buildComplexSqlQuery(parsedQuery, limit, offset); + + logger.debug(`${FILE_NAME}: Executing complex search SQL: ${sqlQuery}`); + logger.debug(`${FILE_NAME}: Query parameters: ${JSON.stringify(params)}`); + + // First get the total count of matching results + const countResult = await new Promise((resolve, reject) => { + db.get(sqlCountQuery, params.slice(0, params.length - 2), (err, row) => { + if (err) { + logger.error(`${FILE_NAME}: Complex search count query error: ${err.message}`); + reject(err); + } else { + resolve(row || { count: 0 }); + } + }); + }); + + const totalCount = countResult.count; + logger.debug(`${FILE_NAME}: Total matching rules for complex query: ${totalCount}`); + + // Now get the actual results with pagination + const results = await new Promise((resolve, reject) => { + db.all(sqlQuery, params, (err, rows) => { + if (err) { + logger.error(`${FILE_NAME}: Complex search query error: ${err.message}`); + reject(err); + } else { + logger.debug(`${FILE_NAME}: Complex search query returned ${rows ? rows.length : 0} results`); + resolve(rows || []); + } + }); + }); + + // Format the results + const formattedResults = results.map(r => ({ + id: r.rule_id, + title: r.title || r.rule_id + })); + + logger.debug(`${FILE_NAME}: Returning ${formattedResults.length} results for complex search`); + + return { + results: formattedResults, + totalCount + }; + } catch (error) { + logger.error(`${FILE_NAME}: Error in complex search operation: ${error.message}`); + logger.debug(`${FILE_NAME}: Complex search error stack: ${error.stack}`); + return { results: [], totalCount: 0 }; + } finally { + // IMPORTANT: Only close the db connection if we're not using FTS + // When using FTS, let searchRulesComplexFTS manage the connection + if (db && !usingFts) { + try { + await new Promise((resolve) => db.close(() => resolve())); + logger.debug(`${FILE_NAME}: Database connection closed after complex search operation`); + } catch (closeError) { + logger.error(`${FILE_NAME}: Error closing database after complex search: ${closeError.message}`); + } + } + } +} + +/** + * Search for Sigma rules using complex query conditions with FTS5 + * Uses the FTS5 virtual table for faster text searching + * + * @param {Object} parsedQuery - The parsed query object + * @param {number} limit - Maximum number of results to return + * @param {number} offset - Number of results to skip (for pagination) + * @param {Object} providedDb - Database connection (optional, will create one if not provided) + * @returns {Promise} Object with results array and total count + */ +async function searchRulesComplexFTS(parsedQuery, limit = 10, offset = 0, providedDb = null) { + if (!parsedQuery || !parsedQuery.valid) { + logger.warn(`${FILE_NAME}: Invalid query object provided for FTS complex search`); + return { results: [], totalCount: 0 }; + } + + logger.info(`${FILE_NAME}: Performing complex FTS search with ${parsedQuery.conditions.length} conditions`); + + let db; + let shouldCloseDb = false; + + try { + // Use provided db connection or create a new one + if (providedDb) { + db = providedDb; + } else { + db = await getDbConnection(); + shouldCloseDb = true; + logger.debug(`${FILE_NAME}: Created new database connection for complex FTS search`); + } + + // Build FTS query from conditions + const { ftsQuery, whereClause, params } = buildComplexFtsQuery(parsedQuery); + + logger.debug(`${FILE_NAME}: FTS query: "${ftsQuery}", additional where: ${whereClause ? whereClause : 'none'}`); + logger.debug(`${FILE_NAME}: Query parameters: ${JSON.stringify(params)}`); + + // Build count query + let countQuery; + let countParams; + + if (whereClause) { + countQuery = ` + SELECT COUNT(*) as count + FROM rule_search + WHERE rule_search MATCH ? + AND ${whereClause} + `; + countParams = [ftsQuery, ...params]; + } else { + countQuery = ` + SELECT COUNT(*) as count + FROM rule_search + WHERE rule_search MATCH ? + `; + countParams = [ftsQuery]; + } + + // Get total count + const countResult = await new Promise((resolve, reject) => { + db.get(countQuery, countParams, (err, row) => { + if (err) { + logger.error(`${FILE_NAME}: Complex FTS count query error: ${err.message}`); + reject(err); + } else { + resolve(row || { count: 0 }); + } + }); + }); + + const totalCount = countResult.count; + logger.debug(`${FILE_NAME}: Total matching rules for complex FTS query: ${totalCount}`); + + // Build results query with pagination + let searchQuery; + let searchParams; + + if (whereClause) { + searchQuery = ` + SELECT rule_id, title + FROM rule_search + WHERE rule_search MATCH ? + AND ${whereClause} + ORDER BY rank + LIMIT ? OFFSET ? + `; + searchParams = [ftsQuery, ...params, limit, offset]; + } else { + searchQuery = ` + SELECT rule_id, title + FROM rule_search + WHERE rule_search MATCH ? + ORDER BY rank + LIMIT ? OFFSET ? + `; + searchParams = [ftsQuery, limit, offset]; + } + + // Get paginated results + const results = await new Promise((resolve, reject) => { + db.all(searchQuery, searchParams, (err, rows) => { + if (err) { + logger.error(`${FILE_NAME}: Complex FTS search query error: ${err.message}`); + reject(err); + } else { + logger.debug(`${FILE_NAME}: Complex FTS search query returned ${rows ? rows.length : 0} results`); + resolve(rows || []); + } + }); + }); + + // Format the results + const formattedResults = results.map(r => ({ + id: r.rule_id, + title: r.title || r.rule_id + })); + + logger.debug(`${FILE_NAME}: Returning ${formattedResults.length} results for complex FTS search`); + + return { + results: formattedResults, + totalCount + }; + } catch (error) { + logger.error(`${FILE_NAME}: Error in complex FTS search operation: ${error.message}`); + logger.debug(`${FILE_NAME}: Complex FTS search error stack: ${error.stack}`); + return { results: [], totalCount: 0 }; + } finally { + // Only close the database if we created it AND we're not in the middle of a transaction + if (db && shouldCloseDb) { + try { + await db.close(); + logger.debug(`${FILE_NAME}: Database connection closed after complex FTS search`); + } catch (closeError) { + logger.error(`${FILE_NAME}: Error closing database after complex FTS search: ${closeError.message}`); + } + } + } +} + +module.exports = { + searchRulesComplex, + searchRulesComplexFTS +}; \ No newline at end of file diff --git a/src/sigma_db/queries/fts-search.js b/src/sigma_db/queries/fts-search.js new file mode 100644 index 0000000..fdaea2b --- /dev/null +++ b/src/sigma_db/queries/fts-search.js @@ -0,0 +1,133 @@ +/** + * fts-search.js + * Functions for Full Text Search (FTS) of Sigma rules + */ + +const { getDbConnection } = require('../sigma_db_connection'); +const logger = require('../../utils/logger'); +const { getFileName } = require('../../utils/file_utils'); +const FILE_NAME = getFileName(__filename); + +/** + * Check if FTS5 virtual table is available + * + * @param {Object} db - Database connection + * @returns {Promise} Whether FTS5 is available + */ +async function checkFtsAvailable(db) { + try { + const result = await new Promise((resolve, reject) => { + db.get("SELECT name FROM sqlite_master WHERE type='table' AND name='rule_search'", (err, row) => { + if (err) { + logger.error(`${FILE_NAME}: Error checking for FTS5 table: ${err.message}`); + reject(err); + } else { + resolve(row !== undefined); + } + }); + }); + + logger.debug(`${FILE_NAME}: FTS5 table availability check: ${result ? 'Available' : 'Not available'}`); + return result; + } catch (error) { + logger.error(`${FILE_NAME}: Error checking FTS availability: ${error.message}`); + return false; + } +} + +/** + * Search for Sigma rules using FTS5 + * Performs a full-text search and returns matching rules with pagination + * + * @param {string} keyword - The keyword to search for + * @param {number} limit - Maximum number of results to return (default: 10) + * @param {number} offset - Number of results to skip (for pagination, default: 0) + * @returns {Promise} Object with results array and total count + */ +async function searchRulesFTS(keyword, limit = 10, offset = 0) { + if (!keyword) { + logger.warn(`${FILE_NAME}: Empty search keyword provided for FTS search`); + return { results: [], totalCount: 0 }; + } + + // Prepare FTS query - add * for prefix matching if not already present + let ftsQuery = keyword.trim(); + if (!ftsQuery.endsWith('*')) { + ftsQuery = `${ftsQuery}*`; + } + + logger.info(`${FILE_NAME}: Performing FTS search with query: "${ftsQuery}" (limit: ${limit}, offset: ${offset})`); + + let db; + try { + db = await getDbConnection(); + logger.debug(`${FILE_NAME}: Database connection established for FTS search`); + + // First get the total count of matching rules + const countQuery = ` + SELECT COUNT(*) as count + FROM rule_search + WHERE rule_search MATCH ? + `; + + const countResult = await new Promise((resolve, reject) => { + db.get(countQuery, [ftsQuery], (err, row) => { + if (err) { + logger.error(`${FILE_NAME}: FTS count query error: ${err.message}`); + reject(err); + } else { + resolve(row || { count: 0 }); + } + }); + }); + + const totalCount = countResult.count; + logger.debug(`${FILE_NAME}: Total matching rules for FTS query "${ftsQuery}": ${totalCount}`); + + // Now get the actual results with pagination + const searchQuery = ` + SELECT rule_id, title + FROM rule_search + WHERE rule_search MATCH ? + ORDER BY rank + LIMIT ? OFFSET ? + `; + + const results = await new Promise((resolve, reject) => { + db.all(searchQuery, [ftsQuery, limit, offset], (err, rows) => { + if (err) { + logger.error(`${FILE_NAME}: FTS search query error: ${err.message}`); + reject(err); + } else { + logger.debug(`${FILE_NAME}: FTS search query returned ${rows ? rows.length : 0} results`); + resolve(rows || []); + } + }); + }); + + logger.debug(`${FILE_NAME}: FTS search results page for query "${ftsQuery}": ${results.length} matches (page ${Math.floor(offset / limit) + 1})`); + + return { + results: results.map(r => ({ id: r.rule_id, title: r.title || r.rule_id })), + totalCount + }; + } catch (error) { + logger.error(`${FILE_NAME}: Error in FTS search operation: ${error.message}`); + logger.debug(`${FILE_NAME}: FTS search error stack: ${error.stack}`); + return { results: [], totalCount: 0 }; + } finally { + if (db) { + try { + await db.close(); + logger.debug(`${FILE_NAME}: Database connection closed after FTS search operation`); + } catch (closeError) { + logger.error(`${FILE_NAME}: Error closing database connection after FTS search: ${closeError.message}`); + } + } + } +} + +module.exports = { + checkFtsAvailable, + searchRulesFTS +}; \ No newline at end of file diff --git a/src/sigma_db/queries/index.js b/src/sigma_db/queries/index.js new file mode 100644 index 0000000..b3e9894 --- /dev/null +++ b/src/sigma_db/queries/index.js @@ -0,0 +1,34 @@ +/** + * index.js + * Central module for accessing all Sigma database query functions + */ + +// Import functions from individual modules +const ruleRetrieval = require('./rule-retrieval'); +const simpleSearch = require('./simple-search'); +const ftsSearch = require('./fts-search'); +const complexSearch = require('./complex-search'); +const statsDebug = require('./stats-debug'); + +// Export all functions +module.exports = { + // Rule retrieval functions + getAllRuleIds: ruleRetrieval.getAllRuleIds, + findRuleById: ruleRetrieval.findRuleById, + + // Search functions + searchRules: simpleSearch.searchRules, + + // FTS search functions + searchRulesFTS: ftsSearch.searchRulesFTS, + checkFtsAvailable: ftsSearch.checkFtsAvailable, + + // Complex search functions + searchRulesComplex: complexSearch.searchRulesComplex, + searchRulesComplexFTS: complexSearch.searchRulesComplexFTS, + + // Stats and debug functions + debugRuleContent: statsDebug.debugRuleContent, + getRuleYamlContent: statsDebug.getRuleYamlContent, + getStatsFromDatabase: statsDebug.getStatsFromDatabase +}; \ No newline at end of file diff --git a/src/sigma_db/queries/query-builders.js b/src/sigma_db/queries/query-builders.js new file mode 100644 index 0000000..0224d38 --- /dev/null +++ b/src/sigma_db/queries/query-builders.js @@ -0,0 +1,258 @@ +/** + * query-builders.js + * Helper functions for building SQL queries + */ + +const logger = require('../../utils/logger'); +const { getFileName } = require('../../utils/file_utils'); +const FILE_NAME = getFileName(__filename); + +/** + * Build the SQL query for complex search based on parsed conditions + * + * @param {Object} parsedQuery - The parsed query object + * @param {number} limit - Results limit + * @param {number} offset - Results offset + * @returns {Object} Object with SQL query, count query, and parameters + */ +function buildComplexSqlQuery(parsedQuery, limit, offset) { + const { conditions, operator } = parsedQuery; + const params = []; + + // Start building the primary table selection + let sqlSelectPart = ` + SELECT DISTINCT r.id as rule_id, + (SELECT param_value FROM rule_parameters WHERE rule_id = r.id AND param_name = 'title' LIMIT 1) as title + FROM sigma_rules r + `; + + // Build WHERE clause based on conditions + let whereClauses = []; + let joinIdx = 0; + + for (const condition of conditions) { + let whereClause = ''; + + switch (condition.field) { + case 'title': + joinIdx++; + whereClause = `EXISTS ( + SELECT 1 FROM rule_parameters p${joinIdx} + WHERE p${joinIdx}.rule_id = r.id + AND p${joinIdx}.param_name = 'title' + AND INSTR(LOWER(p${joinIdx}.param_value), LOWER(?)) > 0 + )`; + params.push(condition.value); + break; + + case 'description': + joinIdx++; + whereClause = `EXISTS ( + SELECT 1 FROM rule_parameters p${joinIdx} + WHERE p${joinIdx}.rule_id = r.id + AND p${joinIdx}.param_name = 'description' + AND INSTR(LOWER(p${joinIdx}.param_value), LOWER(?)) > 0 + )`; + params.push(condition.value); + break; + + case 'logsource': + joinIdx++; + whereClause = `EXISTS ( + SELECT 1 FROM rule_parameters p${joinIdx} + WHERE p${joinIdx}.rule_id = r.id + AND p${joinIdx}.param_name = 'logsource' + AND INSTR(LOWER(p${joinIdx}.param_value), LOWER('"${condition.subfield}":"${condition.value}"')) > 0 + )`; + break; + + case 'tags': + joinIdx++; + whereClause = `EXISTS ( + SELECT 1 FROM rule_parameters p${joinIdx} + WHERE p${joinIdx}.rule_id = r.id + AND p${joinIdx}.param_name = 'tags' + AND INSTR(LOWER(p${joinIdx}.param_value), LOWER(?)) > 0 + )`; + params.push(condition.value); + break; + + case 'date': + joinIdx++; + const dateOperator = condition.operator === 'after' ? '>' : + condition.operator === 'before' ? '<' : '='; + whereClause = `r.date ${dateOperator} date(?)`; + params.push(condition.value); + break; + + case 'author': + joinIdx++; + whereClause = `EXISTS ( + SELECT 1 FROM rule_parameters p${joinIdx} + WHERE p${joinIdx}.rule_id = r.id + AND p${joinIdx}.param_name = 'author' + AND INSTR(LOWER(p${joinIdx}.param_value), LOWER(?)) > 0 + )`; + params.push(condition.value); + break; + + case 'level': + joinIdx++; + whereClause = `EXISTS ( + SELECT 1 FROM rule_parameters p${joinIdx} + WHERE p${joinIdx}.rule_id = r.id + AND p${joinIdx}.param_name = 'level' + AND LOWER(p${joinIdx}.param_value) = LOWER(?) + )`; + params.push(condition.value); + break; + + case 'id': + whereClause = `LOWER(r.id) = LOWER(?)`; + params.push(condition.value); + break; + + case 'keyword': + default: + // Default to searching in title + joinIdx++; + whereClause = `EXISTS ( + SELECT 1 FROM rule_parameters p${joinIdx} + WHERE p${joinIdx}.rule_id = r.id + AND p${joinIdx}.param_name = 'title' + AND INSTR(LOWER(p${joinIdx}.param_value), LOWER(?)) > 0 + )`; + params.push(condition.value); + break; + } + + if (whereClause) { + whereClauses.push(whereClause); + } + } + + // Combine the WHERE clauses with the appropriate operator + let whereStatement = ''; + if (whereClauses.length > 0) { + const combiner = operator === 'AND' ? ' AND ' : ' OR '; + whereStatement = `WHERE ${whereClauses.join(combiner)}`; + } + + // Complete queries + const sqlQuery = ` + ${sqlSelectPart} + ${whereStatement} + ORDER BY rule_id + LIMIT ? OFFSET ? + `; + + const sqlCountQuery = ` + SELECT COUNT(DISTINCT r.id) as count + FROM sigma_rules r + ${whereStatement} + `; + + // Add pagination parameters + params.push(limit); + params.push(offset); + + return { sqlQuery, sqlCountQuery, params }; +} + +/** + * Build FTS query and WHERE clause from parsed query conditions + * + * @param {Object} parsedQuery - The parsed query object + * @returns {Object} Object with FTS query, additional WHERE clause, and parameters + */ +function buildComplexFtsQuery(parsedQuery) { + const { conditions, operator } = parsedQuery; + + // Separate text search conditions from other conditions + const textConditions = []; + const nonTextConditions = []; + + for (const condition of conditions) { + switch (condition.field) { + case 'title': + case 'description': + case 'author': + case 'tags': + case 'keyword': + // These can be handled by FTS directly + textConditions.push(condition); + break; + default: + // These need additional WHERE clauses + nonTextConditions.push(condition); + break; + } + } + + // Build FTS MATCH query + let ftsQueryParts = []; + + for (const condition of textConditions) { + let fieldPrefix = ''; + + // Add field-specific prefix if available + if (condition.field !== 'keyword') { + fieldPrefix = `${condition.field}:`; + } + + // Add wildcard for partial matching if not already present + let value = condition.value.trim(); + if (!value.endsWith('*')) { + value = `${value}*`; + } + + ftsQueryParts.push(`${fieldPrefix}${value}`); + } + + // If no text conditions, use a match-all query + const ftsQuery = ftsQueryParts.length > 0 + ? ftsQueryParts.join(operator === 'AND' ? ' AND ' : ' OR ') + : '*'; + + // Build additional WHERE clauses for non-text conditions + let whereClauseParts = []; + const params = []; + + for (const condition of nonTextConditions) { + switch (condition.field) { + case 'date': + const dateOperator = condition.operator === 'after' ? '>' : + condition.operator === 'before' ? '<' : '='; + whereClauseParts.push(`date ${dateOperator} date(?)`); + params.push(condition.value); + break; + + case 'level': + whereClauseParts.push(`level = ?`); + params.push(condition.value); + break; + + case 'logsource': + whereClauseParts.push(`logsource LIKE ?`); + params.push(`%${condition.subfield}%${condition.value}%`); + break; + + case 'id': + whereClauseParts.push(`rule_id = ?`); + params.push(condition.value); + break; + } + } + + // Combine WHERE clauses + const whereClause = whereClauseParts.length > 0 + ? whereClauseParts.join(operator === 'AND' ? ' AND ' : ' OR ') + : ''; + + return { ftsQuery, whereClause, params }; +} + +module.exports = { + buildComplexSqlQuery, + buildComplexFtsQuery +}; \ No newline at end of file diff --git a/src/sigma_db/queries/rule-retrieval.js b/src/sigma_db/queries/rule-retrieval.js new file mode 100644 index 0000000..10a0a43 --- /dev/null +++ b/src/sigma_db/queries/rule-retrieval.js @@ -0,0 +1,164 @@ +/** + * rule-retrieval.js + * Functions for retrieving Sigma rules and rule IDs + */ + +const { getDbConnection } = require('../sigma_db_connection'); +const logger = require('../../utils/logger'); +const { getFileName } = require('../../utils/file_utils'); +const FILE_NAME = getFileName(__filename); + +/** + * Get a list of all rule IDs in the database + * Useful for bulk operations and database integrity checks + * + * @returns {Promise} Array of rule IDs or empty array on error + */ +async function getAllRuleIds() { + let db; + try { + logger.info(`${FILE_NAME}: Retrieving all rule IDs from database`); + + db = await getDbConnection(); + logger.debug(`${FILE_NAME}: Connected to database for retrieving all rule IDs`); + + const result = await new Promise((resolve, reject) => { + db.all('SELECT id FROM sigma_rules ORDER BY id', [], (err, rows) => { + if (err) { + logger.error(`${FILE_NAME}: Error fetching all rule IDs: ${err.message}`); + reject(err); + } else { + resolve(rows || []); + } + }); + }); + + logger.debug(`${FILE_NAME}: Retrieved ${result.length} rule IDs from database`); + return result.map(row => row.id); + } catch (error) { + logger.error(`${FILE_NAME}: Error retrieving all rule IDs: ${error.message}`); + logger.debug(`${FILE_NAME}: Error stack: ${error.stack}`); + return []; + } finally { + if (db) { + try { + await db.close(); + logger.debug(`${FILE_NAME}: Database connection closed after retrieving all rule IDs`); + } catch (closeError) { + logger.warn(`${FILE_NAME}: Error closing database: ${closeError.message}`); + } + } + } +} + +/** + * Find a Sigma rule by its ID + * Retrieves rule data and associated parameters from the database + * + * @param {string} ruleId - The ID of the rule to find + * @returns {Promise} The rule object or null if not found + */ +async function findRuleById(ruleId) { + if (!ruleId) { + logger.warn(`${FILE_NAME}: Cannot find rule: Missing rule ID`); + return null; + } + + let db; + try { + db = await getDbConnection(); + logger.debug(`${FILE_NAME}: Connected to database for rule lookup: ${ruleId}`); + + // Get the base rule using promisified method + const rule = await db.getAsync('SELECT * FROM sigma_rules WHERE id = ?', [ruleId]); + if (!rule) { + logger.warn(`${FILE_NAME}: Rule with ID ${ruleId} not found in database`); + return null; + } + + logger.debug(`${FILE_NAME}: Found base rule with ID ${ruleId}, content length: ${rule.content ? rule.content.length : 0}`); + + // Get parameters using promisified method + const paramsAsync = await db.allAsync('SELECT param_name, param_value, param_type FROM rule_parameters WHERE rule_id = ?', [ruleId]); + logger.debug(`${FILE_NAME}: Params query returned ${paramsAsync ? paramsAsync.length : 0} results via allAsync`); + + // Check if content is missing + if (!rule.content) { + logger.warn(`${FILE_NAME}: Rule with ID ${ruleId} exists but has no content`); + rule.content_missing = true; + } + + // Get all parameters for this rule with case-insensitive matching + try { + const params = await new Promise((resolve, reject) => { + db.all( + 'SELECT param_name, param_value, param_type FROM rule_parameters WHERE LOWER(rule_id) = LOWER(?)', + [ruleId], + (err, rows) => { + if (err) reject(err); + else resolve(rows); + } + ); + }); + + logger.debug(`${FILE_NAME}: Retrieved ${params ? params.length : 0} parameters for rule ${ruleId}`); + + // Validate params is an array + if (params && Array.isArray(params)) { + // Attach parameters to the rule object + rule.parameters = {}; + + for (const param of params) { + if (param && param.param_name) { + // Convert value based on type + let value = param.param_value; + + if (param.param_type === 'object' || param.param_type === 'array') { + try { + value = JSON.parse(param.param_value); + } catch (parseError) { + logger.warn(`${FILE_NAME}: Failed to parse JSON for parameter ${param.param_name}: ${parseError.message}`); + } + } else if (param.param_type === 'boolean') { + value = param.param_value === 'true'; + } else if (param.param_type === 'number') { + value = Number(param.param_value); + } + + rule.parameters[param.param_name] = value; + } + } + + logger.debug(`${FILE_NAME}: Successfully processed ${Object.keys(rule.parameters).length} parameters for rule ${ruleId}`); + } else { + logger.warn(`${FILE_NAME}: Parameters for rule ${ruleId} not available or not iterable`); + rule.parameters = {}; + } + } catch (paramError) { + logger.error(`${FILE_NAME}: Error fetching parameters for rule ${ruleId}: ${paramError.message}`); + logger.debug(`${FILE_NAME}: Parameter error stack: ${paramError.stack}`); + rule.parameters = {}; + } + + return rule; + } catch (error) { + logger.error(`${FILE_NAME}: Error finding rule ${ruleId}: ${error.message}`); + logger.debug(`${FILE_NAME}: Error stack: ${error.stack}`); + return null; + } finally { + // Close the database connection if it was opened + if (db && typeof db.close === 'function') { + try { + await db.close(); + logger.debug(`${FILE_NAME}: Database connection closed after rule lookup`); + } catch (closeError) { + logger.warn(`${FILE_NAME}: Error closing database connection: ${closeError.message}`); + } + } + } +} + +module.exports = { + getAllRuleIds, + findRuleById +}; \ No newline at end of file diff --git a/src/sigma_db/queries/simple-search.js b/src/sigma_db/queries/simple-search.js new file mode 100644 index 0000000..453a16d --- /dev/null +++ b/src/sigma_db/queries/simple-search.js @@ -0,0 +1,118 @@ +/** + * simple-search.js + * Functions for basic search of Sigma rules + */ + +const { getDbConnection } = require('../sigma_db_connection'); +const logger = require('../../utils/logger'); +const { getFileName } = require('../../utils/file_utils'); +const FILE_NAME = getFileName(__filename); + +// Import FTS functions - need to use relative path for proper circular dependency handling +const { checkFtsAvailable, searchRulesFTS } = require('./fts-search'); + +/** + * Search for Sigma rules by keyword in rule titles + * Performs a case-insensitive search and returns matching rules with pagination + * + * @param {string} keyword - The keyword to search for + * @param {number} limit - Maximum number of results to return (default: 10) + * @param {number} offset - Number of results to skip (for pagination, default: 0) + * @returns {Promise} Object with results array and total count + */ +async function searchRules(keyword, limit = 10, offset = 0) { + if (!keyword) { + logger.warn(`${FILE_NAME}: Empty search keyword provided`); + return { results: [], totalCount: 0 }; + } + + // Sanitize keyword to prevent SQL injection + const sanitizedKeyword = keyword.replace(/'/g, "''"); + logger.info(`${FILE_NAME}: Searching for rules with keyword in title: ${sanitizedKeyword} (limit: ${limit}, offset: ${offset})`); + + let db; + try { + // Make sure we properly await the DB connection + db = await getDbConnection(); + logger.debug(`${FILE_NAME}: Database connection established for search`); + + // Use FTS5 for faster searching if available + const ftsAvailable = await checkFtsAvailable(db); + + if (ftsAvailable) { + logger.debug(`${FILE_NAME}: Using FTS5 for keyword search`); + return searchRulesFTS(keyword, limit, offset); + } + + // If FTS5 is not available, use the legacy search method + logger.debug(`${FILE_NAME}: FTS5 not available, using legacy search method`); + + // First get the total count of matching rules (for pagination info) + const countQuery = ` + SELECT COUNT(*) as count + FROM rule_parameters + WHERE param_name = 'title' + AND INSTR(LOWER(param_value), LOWER(?)) > 0 + `; + + const countResult = await new Promise((resolve, reject) => { + db.get(countQuery, [sanitizedKeyword], (err, row) => { + if (err) { + logger.error(`${FILE_NAME}: Count query error: ${err.message}`); + reject(err); + } else { + resolve(row || { count: 0 }); + } + }); + }); + + const totalCount = countResult.count; + logger.debug(`${FILE_NAME}: Total matching rules for "${sanitizedKeyword}": ${totalCount}`); + + // Use parameterized query instead of string interpolation for better security + const instrQuery = ` + SELECT rule_id, param_value AS title + FROM rule_parameters + WHERE param_name = 'title' + AND INSTR(LOWER(param_value), LOWER(?)) > 0 + LIMIT ? OFFSET ? + `; + + const results = await new Promise((resolve, reject) => { + db.all(instrQuery, [sanitizedKeyword, limit, offset], (err, rows) => { + if (err) { + logger.error(`${FILE_NAME}: Search query error: ${err.message}`); + reject(err); + } else { + logger.debug(`${FILE_NAME}: Search query returned ${rows ? rows.length : 0} results`); + resolve(rows || []); + } + }); + }); + + logger.debug(`${FILE_NAME}: Search results page for keyword "${sanitizedKeyword}": ${results.length} matches (page ${Math.floor(offset / limit) + 1})`); + + return { + results: results.map(r => ({ id: r.rule_id, title: r.title })), + totalCount + }; + } catch (error) { + logger.error(`${FILE_NAME}: Error in search operation: ${error.message}`); + logger.debug(`${FILE_NAME}: Search error stack: ${error.stack}`); + return { results: [], totalCount: 0 }; + } finally { + // Make sure we properly close the connection + if (db) { + try { + await new Promise((resolve) => db.close(() => resolve())); + logger.debug(`${FILE_NAME}: Database connection closed after search operation`); + } catch (closeError) { + logger.error(`${FILE_NAME}: Error closing database connection after search: ${closeError.message}`); + } + } + } +} + +module.exports = { + searchRules +}; \ No newline at end of file diff --git a/src/sigma_db/queries/stats-debug.js b/src/sigma_db/queries/stats-debug.js new file mode 100644 index 0000000..e8fda9c --- /dev/null +++ b/src/sigma_db/queries/stats-debug.js @@ -0,0 +1,328 @@ +/** + * stats-debug.js + * Functions for database statistics and debugging + */ + +const { getDbConnection } = require('../sigma_db_connection'); +const logger = require('../../utils/logger'); +const { DB_PATH } = require('../../config/appConfig'); +const path = require('path'); +const { getFileName } = require('../../utils/file_utils'); +const FILE_NAME = getFileName(__filename); + +/** + * Debug function to retrieve detailed information about a rule's content + * Useful for diagnosing issues with rule retrieval and content parsing + * + * @param {string} ruleId - The ID of the rule to debug + * @returns {Promise} Object containing debug information or null on error + */ +async function debugRuleContent(ruleId) { + if (!ruleId) { + logger.warn(`${FILE_NAME}: Cannot debug rule: Missing rule ID`); + return null; + } + + let db; + try { + db = await getDbConnection(); + + const absolutePath = path.resolve(DB_PATH); + logger.debug(`${FILE_NAME}: Debug function connecting to DB at path: ${absolutePath}`); + + // Get raw rule record + const rule = await db.get('SELECT id, file_path, length(content) as content_length, typeof(content) as content_type FROM sigma_rules WHERE id = ?', [ruleId]); + + if (!rule) { + logger.warn(`${FILE_NAME}: Rule with ID ${ruleId} not found during debug operation`); + return { error: 'Rule not found', ruleId }; + } + + // Return just the rule information without the undefined variables + return { + rule, + ruleId + }; + } catch (error) { + logger.error(`${FILE_NAME}: Debug error for rule ${ruleId}: ${error.message}`); + logger.debug(`${FILE_NAME}: Debug error stack: ${error.stack}`); + return { + error: error.message, + stack: error.stack, + ruleId + }; + } finally { + if (db) { + try { + await db.close(); + logger.debug(`${FILE_NAME}: Database connection closed after debug operation`); + } catch (closeError) { + logger.warn(`${FILE_NAME}: Error closing database after debug: ${closeError.message}`); + } + } + } +} + +/** + * Get the raw YAML content of a Sigma rule + * Retrieves the content field from the database which should contain YAML + * + * @param {string} ruleId - The ID of the rule + * @returns {Promise} Object with success flag and content or error message + */ +async function getRuleYamlContent(ruleId) { + if (!ruleId) { + logger.warn(`${FILE_NAME}: Cannot get YAML content: Missing rule ID`); + return { success: false, message: 'Missing rule ID' }; + } + + let db; + try { + logger.info(`${FILE_NAME}: Fetching YAML content for rule: ${ruleId}`); + logger.debug(`${FILE_NAME}: Rule ID type: ${typeof ruleId}, length: ${ruleId.length}`); + + db = await getDbConnection(); + logger.debug(`${FILE_NAME}: Connected to database for YAML retrieval`); + + // Debug query before execution + const debugResult = await db.get('SELECT id, typeof(content) as content_type, length(content) as content_length FROM sigma_rules WHERE id = ?', [ruleId]); + logger.debug(`${FILE_NAME}: Debug query result: ${JSON.stringify(debugResult || 'not found')}`); + + if (!debugResult) { + logger.warn(`${FILE_NAME}: Rule with ID ${ruleId} not found in debug query`); + return { success: false, message: 'Rule not found' }; + } + + // Get actual content + const rule = await new Promise((resolve, reject) => { + db.get('SELECT content FROM sigma_rules WHERE id = ?', [ruleId], (err, row) => { + if (err) { + logger.error(`${FILE_NAME}: Content query error: ${err.message}`); + reject(err); + } else { + resolve(row || null); + } + }); + }); + logger.debug(`${FILE_NAME}: Content query result for ${ruleId}: ${rule ? 'Found' : 'Not found'}`); + + if (!rule) { + logger.warn(`${FILE_NAME}: Rule with ID ${ruleId} not found in content query`); + return { success: false, message: 'Rule not found' }; + } + + if (!rule.content) { + logger.warn(`${FILE_NAME}: Rule with ID ${ruleId} found but has no content`); + return { success: false, message: 'Rule found but content is empty' }; + } + + logger.debug(`${FILE_NAME}: Content retrieved successfully for ${ruleId}, type: ${typeof rule.content}, length: ${rule.content.length}`); + + return { success: true, content: rule.content }; + } catch (error) { + logger.error(`${FILE_NAME}: Error retrieving YAML content for ${ruleId}: ${error.message}`); + logger.debug(`${FILE_NAME}: YAML retrieval error stack: ${error.stack}`); + return { success: false, message: `Error retrieving YAML: ${error.message}` }; + } finally { + if (db) { + try { + await db.close(); + logger.debug(`${FILE_NAME}: Database connection closed after YAML retrieval`); + } catch (closeError) { + logger.warn(`${FILE_NAME}: Error closing database after YAML retrieval: ${closeError.message}`); + } + } + } +} + +/** + * Get statistics about Sigma rules in the database + * Collects counts, categories, and other aggregate information + * + * @returns {Promise} Object with various statistics about the rules + */ +async function getStatsFromDatabase() { + let db; + try { + db = await getDbConnection(); + logger.debug(`${FILE_NAME}: Connected to database for statistics`); + + // Get total rule count + const totalRulesResult = await new Promise((resolve, reject) => { + db.get('SELECT COUNT(*) as count FROM sigma_rules', (err, row) => { + if (err) reject(err); + else resolve(row); + }); + }); + const totalRules = totalRulesResult.count; + + // Get last update time + const lastUpdateResult = await new Promise((resolve, reject) => { + db.get('SELECT MAX(date) as last_update FROM sigma_rules', (err, row) => { + if (err) reject(err); + else resolve(row); + }); + }); + const lastUpdate = lastUpdateResult.last_update; + + // Get rules by log source count (Windows, Linux, macOS) + const windowsRulesResult = await new Promise((resolve, reject) => { + db.get(` + SELECT COUNT(DISTINCT rule_id) as count + FROM rule_parameters + WHERE param_name = 'logsource' AND + param_value LIKE '%"product":"windows"%'`, + (err, row) => { + if (err) reject(err); + else resolve(row); + }); + }); + const windowsRules = windowsRulesResult.count || 0; + + const linuxRulesResult = await new Promise((resolve, reject) => { + db.get(` + SELECT COUNT(DISTINCT rule_id) as count + FROM rule_parameters + WHERE param_name = 'logsource' AND + param_value LIKE '%"product":"linux"%'`, + (err, row) => { + if (err) reject(err); + else resolve(row); + }); + }); + const linuxRules = linuxRulesResult.count || 0; + + const macosRulesResult = await new Promise((resolve, reject) => { + db.get(` + SELECT COUNT(DISTINCT rule_id) as count + FROM rule_parameters + WHERE param_name = 'logsource' AND + param_value LIKE '%"product":"macos"%'`, + (err, row) => { + if (err) reject(err); + else resolve(row); + }); + }); + const macosRules = macosRulesResult.count || 0; + + // Get rules by severity level + const severityStats = await new Promise((resolve, reject) => { + db.all(` + SELECT param_value AS level, COUNT(DISTINCT rule_id) as count + FROM rule_parameters + WHERE param_name = 'level' + GROUP BY param_value + ORDER BY + CASE + WHEN param_value = 'critical' THEN 1 + WHEN param_value = 'high' THEN 2 + WHEN param_value = 'medium' THEN 3 + WHEN param_value = 'low' THEN 4 + WHEN param_value = 'informational' THEN 5 + ELSE 6 + END`, + (err, rows) => { + if (err) reject(err); + else resolve(rows); + }); + }); + + // Get top 5 rule authors + const topAuthors = await new Promise((resolve, reject) => { + db.all(` + SELECT param_value AS author, COUNT(*) as count + FROM rule_parameters + WHERE param_name = 'author' + GROUP BY param_value + ORDER BY count DESC + LIMIT 5`, + (err, rows) => { + if (err) reject(err); + else resolve(rows); + }); + }); + + // Get empty content count (rules with missing YAML) + const emptyContentResult = await new Promise((resolve, reject) => { + db.get(` + SELECT COUNT(*) as count + FROM sigma_rules + WHERE content IS NULL OR content = ''`, + (err, row) => { + if (err) reject(err); + else resolve(row); + }); + }); + const emptyContentCount = emptyContentResult.count; + + // Get MITRE ATT&CK tactics statistics + const mitreStats = await new Promise((resolve, reject) => { + db.all(` + SELECT param_value AS tag, COUNT(DISTINCT rule_id) as count + FROM rule_parameters + WHERE param_name = 'tags' AND param_value LIKE 'attack.%' + GROUP BY param_value + ORDER BY count DESC + LIMIT 10`, + (err, rows) => { + if (err) reject(err); + else resolve(rows); + }); + }); + + // Format MITRE tactics for display + const formattedMitreTactics = mitreStats.map(item => { + const tactic = item.tag.substring(7); // Remove 'attack.' prefix + return { + tactic: tactic, + count: item.count + }; + }); + + // Compile all statistics + const stats = { + totalRules, + lastUpdate, + operatingSystems: { + windows: windowsRules, + linux: linuxRules, + macos: macosRules, + other: totalRules - (windowsRules + linuxRules + macosRules) + }, + severityLevels: severityStats.map(s => ({ level: s.level, count: s.count })), + topAuthors: topAuthors.map(a => ({ name: a.author, count: a.count })), + databaseHealth: { + emptyContentCount, + contentPercentage: totalRules > 0 ? Math.round(((totalRules - emptyContentCount) / totalRules) * 100) : 0 + }, + mitreTactics: formattedMitreTactics + }; + + return { + success: true, + stats + }; + } catch (error) { + logger.error(`${FILE_NAME}: Error retrieving statistics: ${error.message}`); + logger.debug(`${FILE_NAME}: Error stack: ${error.stack}`); + return { + success: false, + message: `Error retrieving statistics: ${error.message}` + }; + } finally { + if (db) { + try { + await new Promise((resolve) => db.close(() => resolve())); + logger.debug(`${FILE_NAME}: Database connection closed after statistics retrieval`); + } catch (closeError) { + logger.warn(`${FILE_NAME}: Error closing database: ${closeError.message}`); + } + } + } +} + +module.exports = { + debugRuleContent, + getRuleYamlContent, + getStatsFromDatabase +}; \ No newline at end of file diff --git a/src/sigma_db/sigma_db_queries.js b/src/sigma_db/sigma_db_queries.js deleted file mode 100644 index 8fa53f9..0000000 --- a/src/sigma_db/sigma_db_queries.js +++ /dev/null @@ -1,1207 +0,0 @@ -/** - * - * sigma_db_queries.js - * this script contains functions to interact with the Sigma database - * - * IMPORTANT: - * SQLite queries need explicit Promise handling when using db.all() - * - * We had an issue in that the Promise returned by db.all() wasn't being - * properly resolved in the async context. By wrapping the db.all() call in - * a new Promise and explicitly handling the callback, we ensure the query - * completes before continuing. This is important with SQLite where the - * connection state management can sometimes be tricky with async/await. - * - */ -const { getDbConnection } = require('./sigma_db_connection'); -const logger = require('../utils/logger'); -const { DB_PATH } = require('../config/appConfig'); -const path = require('path'); - -const { getFileName } = require('../utils/file_utils'); -const FILE_NAME = getFileName(__filename); - - -/** - * Get a list of all rule IDs in the database - * Useful for bulk operations and database integrity checks - * - * @returns {Promise} Array of rule IDs or empty array on error - */ -async function getAllRuleIds() { - let db; - try { - logger.info(`${FILE_NAME}: Retrieving all rule IDs from database`); - - db = await getDbConnection(); - logger.debug(`${FILE_NAME}: Connected to database for retrieving all rule IDs`); - - const result = await new Promise((resolve, reject) => { - db.all('SELECT id FROM sigma_rules ORDER BY id', [], (err, rows) => { - if (err) { - logger.error(`${FILE_NAME}: Error fetching all rule IDs: ${err.message}`); - reject(err); - } else { - resolve(rows || []); - } - }); - }); - - logger.debug(`${FILE_NAME}: Retrieved ${result.length} rule IDs from database`); - return result.map(row => row.id); - } catch (error) { - logger.error(`${FILE_NAME}: Error retrieving all rule IDs: ${error.message}`); - logger.debug(`${FILE_NAME}: Error stack: ${error.stack}`); - return []; - } finally { - if (db) { - try { - await db.close(); - logger.debug(`${FILE_NAME}: Database connection closed after retrieving all rule IDs`); - } catch (closeError) { - logger.warn(`${FILE_NAME}: Error closing database: ${closeError.message}`); - } - } - } -} - - -/** - * Find a Sigma rule by its ID - * Retrieves rule data and associated parameters from the database - * - * @param {string} ruleId - The ID of the rule to find - * @returns {Promise} The rule object or null if not found - */ -async function findRuleById(ruleId) { - if (!ruleId) { - logger.warn(`${FILE_NAME}: Cannot find rule: Missing rule ID`); - return null; - } - - let db; - try { - db = await getDbConnection(); - logger.debug(`${FILE_NAME}: Connected to database for rule lookup: ${ruleId}`); - - // Get the base rule using promisified method - const rule = await db.getAsync('SELECT * FROM sigma_rules WHERE id = ?', [ruleId]); - if (!rule) { - logger.warn(`${FILE_NAME}: Rule with ID ${ruleId} not found in database`); - return null; - } - - logger.debug(`${FILE_NAME}: Found base rule with ID ${ruleId}, content length: ${rule.content ? rule.content.length : 0}`); - - // Get parameters using promisified method - const paramsAsync = await db.allAsync('SELECT param_name, param_value, param_type FROM rule_parameters WHERE rule_id = ?', [ruleId]); - logger.debug(`${FILE_NAME}: Params query returned ${paramsAsync ? paramsAsync.length : 0} results via allAsync`); - - // Check if content is missing - if (!rule.content) { - logger.warn(`${FILE_NAME}: Rule with ID ${ruleId} exists but has no content`); - rule.content_missing = true; - } - - // Get all parameters for this rule with case-insensitive matching - try { - const params = await new Promise((resolve, reject) => { - db.all( - 'SELECT param_name, param_value, param_type FROM rule_parameters WHERE LOWER(rule_id) = LOWER(?)', - [ruleId], - (err, rows) => { - if (err) reject(err); - else resolve(rows); - } - ); - }); - - logger.debug(`${FILE_NAME}: Retrieved ${params ? params.length : 0} parameters for rule ${ruleId}`); - - // Validate params is an array - if (params && Array.isArray(params)) { - // Attach parameters to the rule object - rule.parameters = {}; - - for (const param of params) { - if (param && param.param_name) { - // Convert value based on type - let value = param.param_value; - - if (param.param_type === 'object' || param.param_type === 'array') { - try { - value = JSON.parse(param.param_value); - } catch (parseError) { - logger.warn(`${FILE_NAME}: Failed to parse JSON for parameter ${param.param_name}: ${parseError.message}`); - } - } else if (param.param_type === 'boolean') { - value = param.param_value === 'true'; - } else if (param.param_type === 'number') { - value = Number(param.param_value); - } - - rule.parameters[param.param_name] = value; - } - } - - logger.debug(`${FILE_NAME}: Successfully processed ${Object.keys(rule.parameters).length} parameters for rule ${ruleId}`); - } else { - logger.warn(`${FILE_NAME}: Parameters for rule ${ruleId} not available or not iterable`); - rule.parameters = {}; - } - } catch (paramError) { - logger.error(`${FILE_NAME}: Error fetching parameters for rule ${ruleId}: ${paramError.message}`); - logger.debug(`${FILE_NAME}: Parameter error stack: ${paramError.stack}`); - rule.parameters = {}; - } - - return rule; - } catch (error) { - logger.error(`${FILE_NAME}: Error finding rule ${ruleId}: ${error.message}`); - logger.debug(`${FILE_NAME}: Error stack: ${error.stack}`); - return null; - } finally { - // Close the database connection if it was opened - if (db && typeof db.close === 'function') { - try { - await db.close(); - logger.debug(`${FILE_NAME}: Database connection closed after rule lookup`); - } catch (closeError) { - logger.warn(`${FILE_NAME}: Error closing database connection: ${closeError.message}`); - } - } - } -} - -/** - * Search for Sigma rules by keyword in rule titles - * Performs a case-insensitive search and returns matching rules with pagination - * - * @param {string} keyword - The keyword to search for - * @param {number} limit - Maximum number of results to return (default: 10) - * @param {number} offset - Number of results to skip (for pagination, default: 0) - * @returns {Promise} Object with results array and total count - */ -async function searchRules(keyword, limit = 10, offset = 0) { - if (!keyword) { - logger.warn(`${FILE_NAME}: Empty search keyword provided`); - return { results: [], totalCount: 0 }; - } - - // Sanitize keyword to prevent SQL injection - const sanitizedKeyword = keyword.replace(/'/g, "''"); - logger.info(`${FILE_NAME}: Searching for rules with keyword in title: ${sanitizedKeyword} (limit: ${limit}, offset: ${offset})`); - - let db; - try { - // Make sure we properly await the DB connection - db = await getDbConnection(); - logger.debug(`${FILE_NAME}: Database connection established for search`); - - // Use FTS5 for faster searching if available - const ftsAvailable = await checkFtsAvailable(db); - - if (ftsAvailable) { - logger.debug(`${FILE_NAME}: Using FTS5 for keyword search`); - return searchRulesFTS(keyword, limit, offset); - } - - // If FTS5 is not available, use the legacy search method - logger.debug(`${FILE_NAME}: FTS5 not available, using legacy search method`); - - // First get the total count of matching rules (for pagination info) - const countQuery = ` - SELECT COUNT(*) as count - FROM rule_parameters - WHERE param_name = 'title' - AND INSTR(LOWER(param_value), LOWER(?)) > 0 - `; - - const countResult = await new Promise((resolve, reject) => { - db.get(countQuery, [sanitizedKeyword], (err, row) => { - if (err) { - logger.error(`${FILE_NAME}: Count query error: ${err.message}`); - reject(err); - } else { - resolve(row || { count: 0 }); - } - }); - }); - - const totalCount = countResult.count; - logger.debug(`${FILE_NAME}: Total matching rules for "${sanitizedKeyword}": ${totalCount}`); - - // Use parameterized query instead of string interpolation for better security - const instrQuery = ` - SELECT rule_id, param_value AS title - FROM rule_parameters - WHERE param_name = 'title' - AND INSTR(LOWER(param_value), LOWER(?)) > 0 - LIMIT ? OFFSET ? - `; - - const results = await new Promise((resolve, reject) => { - db.all(instrQuery, [sanitizedKeyword, limit, offset], (err, rows) => { - if (err) { - logger.error(`${FILE_NAME}: Search query error: ${err.message}`); - reject(err); - } else { - logger.debug(`${FILE_NAME}: Search query returned ${rows ? rows.length : 0} results`); - resolve(rows || []); - } - }); - }); - - logger.debug(`${FILE_NAME}: Search results page for keyword "${sanitizedKeyword}": ${results.length} matches (page ${Math.floor(offset / limit) + 1})`); - - return { - results: results.map(r => ({ id: r.rule_id, title: r.title })), - totalCount - }; - } catch (error) { - logger.error(`${FILE_NAME}: Error in search operation: ${error.message}`); - logger.debug(`${FILE_NAME}: Search error stack: ${error.stack}`); - return { results: [], totalCount: 0 }; - } finally { - // Make sure we properly close the connection - if (db) { - try { - await new Promise((resolve) => db.close(() => resolve())); - logger.debug(`${FILE_NAME}: Database connection closed after search operation`); - } catch (closeError) { - logger.error(`${FILE_NAME}: Error closing database connection after search: ${closeError.message}`); - } - } - } -} - -/** - * Check if FTS5 virtual table is available - * - * @param {Object} db - Database connection - * @returns {Promise} Whether FTS5 is available - */ -async function checkFtsAvailable(db) { - try { - const result = await new Promise((resolve, reject) => { - db.get("SELECT name FROM sqlite_master WHERE type='table' AND name='rule_search'", (err, row) => { - if (err) { - logger.error(`${FILE_NAME}: Error checking for FTS5 table: ${err.message}`); - reject(err); - } else { - resolve(row !== undefined); - } - }); - }); - - logger.debug(`${FILE_NAME}: FTS5 table availability check: ${result ? 'Available' : 'Not available'}`); - return result; - } catch (error) { - logger.error(`${FILE_NAME}: Error checking FTS availability: ${error.message}`); - return false; - } -} - -/** - * Search for Sigma rules using FTS5 - * Performs a full-text search and returns matching rules with pagination - * - * @param {string} keyword - The keyword to search for - * @param {number} limit - Maximum number of results to return (default: 10) - * @param {number} offset - Number of results to skip (for pagination, default: 0) - * @returns {Promise} Object with results array and total count - */ -async function searchRulesFTS(keyword, limit = 10, offset = 0) { - if (!keyword) { - logger.warn(`${FILE_NAME}: Empty search keyword provided for FTS search`); - return { results: [], totalCount: 0 }; - } - - // Prepare FTS query - add * for prefix matching if not already present - let ftsQuery = keyword.trim(); - if (!ftsQuery.endsWith('*')) { - ftsQuery = `${ftsQuery}*`; - } - - logger.info(`${FILE_NAME}: Performing FTS search with query: "${ftsQuery}" (limit: ${limit}, offset: ${offset})`); - - let db; - try { - db = await getDbConnection(); - logger.debug(`${FILE_NAME}: Database connection established for FTS search`); - - // First get the total count of matching rules - const countQuery = ` - SELECT COUNT(*) as count - FROM rule_search - WHERE rule_search MATCH ? - `; - - const countResult = await new Promise((resolve, reject) => { - db.get(countQuery, [ftsQuery], (err, row) => { - if (err) { - logger.error(`${FILE_NAME}: FTS count query error: ${err.message}`); - reject(err); - } else { - resolve(row || { count: 0 }); - } - }); - }); - - const totalCount = countResult.count; - logger.debug(`${FILE_NAME}: Total matching rules for FTS query "${ftsQuery}": ${totalCount}`); - - // Now get the actual results with pagination - const searchQuery = ` - SELECT rule_id, title - FROM rule_search - WHERE rule_search MATCH ? - ORDER BY rank - LIMIT ? OFFSET ? - `; - - const results = await new Promise((resolve, reject) => { - db.all(searchQuery, [ftsQuery, limit, offset], (err, rows) => { - if (err) { - logger.error(`${FILE_NAME}: FTS search query error: ${err.message}`); - reject(err); - } else { - logger.debug(`${FILE_NAME}: FTS search query returned ${rows ? rows.length : 0} results`); - resolve(rows || []); - } - }); - }); - - logger.debug(`${FILE_NAME}: FTS search results page for query "${ftsQuery}": ${results.length} matches (page ${Math.floor(offset / limit) + 1})`); - - return { - results: results.map(r => ({ id: r.rule_id, title: r.title || r.rule_id })), - totalCount - }; - } catch (error) { - logger.error(`${FILE_NAME}: Error in FTS search operation: ${error.message}`); - logger.debug(`${FILE_NAME}: FTS search error stack: ${error.stack}`); - return { results: [], totalCount: 0 }; - } finally { - if (db) { - try { - await db.close(); - logger.debug(`${FILE_NAME}: Database connection closed after FTS search operation`); - } catch (closeError) { - logger.error(`${FILE_NAME}: Error closing database connection after FTS search: ${closeError.message}`); - } - } - } -} - -/** - * Search for Sigma rules using complex query conditions - * Supports filtering by multiple attributes like title, logsource, tags, etc. - * - * @param {Object} parsedQuery - The parsed query object containing conditions and operator - * @param {number} limit - Maximum number of results to return - * @param {number} offset - Number of results to skip (for pagination) - * @returns {Promise} Object with results array and total count - */ -async function searchRulesComplex(parsedQuery, limit = 10, offset = 0) { - if (!parsedQuery || !parsedQuery.valid) { - logger.warn(`${FILE_NAME}: Invalid query object provided`); - return { results: [], totalCount: 0 }; - } - - logger.info(`${FILE_NAME}: Performing complex search with ${parsedQuery.conditions.length} conditions (limit: ${limit}, offset: ${offset})`); - - let db; - // Declare this at function scope so it's available in the finally block - let usingFts = false; - - try { - db = await getDbConnection(); - logger.debug(`${FILE_NAME}: Database connection established for complex search`); - - // Check if FTS5 is available - const ftsAvailable = await checkFtsAvailable(db); - - if (ftsAvailable) { - logger.debug(`${FILE_NAME}: Using FTS5 for complex search`); - // Set flag that we're using FTS - usingFts = true; - // Pass db connection to searchRulesComplexFTS and let that function manage it - const results = await searchRulesComplexFTS(parsedQuery, limit, offset, db); - return results; - } - - logger.debug(`${FILE_NAME}: FTS5 not available, using legacy complex search method`); - - // Build the SQL query based on the conditions - const { sqlQuery, sqlCountQuery, params } = buildComplexSqlQuery(parsedQuery, limit, offset); - - logger.debug(`${FILE_NAME}: Executing complex search SQL: ${sqlQuery}`); - logger.debug(`${FILE_NAME}: Query parameters: ${JSON.stringify(params)}`); - - // First get the total count of matching results - const countResult = await new Promise((resolve, reject) => { - db.get(sqlCountQuery, params.slice(0, params.length - 2), (err, row) => { - if (err) { - logger.error(`${FILE_NAME}: Complex search count query error: ${err.message}`); - reject(err); - } else { - resolve(row || { count: 0 }); - } - }); - }); - - const totalCount = countResult.count; - logger.debug(`${FILE_NAME}: Total matching rules for complex query: ${totalCount}`); - - // Now get the actual results with pagination - const results = await new Promise((resolve, reject) => { - db.all(sqlQuery, params, (err, rows) => { - if (err) { - logger.error(`${FILE_NAME}: Complex search query error: ${err.message}`); - reject(err); - } else { - logger.debug(`${FILE_NAME}: Complex search query returned ${rows ? rows.length : 0} results`); - resolve(rows || []); - } - }); - }); - - // Format the results - const formattedResults = results.map(r => ({ - id: r.rule_id, - title: r.title || r.rule_id - })); - - logger.debug(`${FILE_NAME}: Returning ${formattedResults.length} results for complex search`); - - return { - results: formattedResults, - totalCount - }; - } catch (error) { - logger.error(`${FILE_NAME}: Error in complex search operation: ${error.message}`); - logger.debug(`${FILE_NAME}: Complex search error stack: ${error.stack}`); - return { results: [], totalCount: 0 }; - } finally { - // IMPORTANT: Only close the db connection if we're not using FTS - // When using FTS, let searchRulesComplexFTS manage the connection - if (db && !usingFts) { - try { - await new Promise((resolve) => db.close(() => resolve())); - logger.debug(`${FILE_NAME}: Database connection closed after complex search operation`); - } catch (closeError) { - logger.error(`${FILE_NAME}: Error closing database after complex search: ${closeError.message}`); - } - } - } -} - -/** - * Search for Sigma rules using complex query conditions with FTS5 - * Uses the FTS5 virtual table for faster text searching - * - * @param {Object} parsedQuery - The parsed query object - * @param {number} limit - Maximum number of results to return - * @param {number} offset - Number of results to skip (for pagination) - * @param {Object} providedDb - Database connection (optional, will create one if not provided) - * @returns {Promise} Object with results array and total count - */ -async function searchRulesComplexFTS(parsedQuery, limit = 10, offset = 0, providedDb = null) { - if (!parsedQuery || !parsedQuery.valid) { - logger.warn(`${FILE_NAME}: Invalid query object provided for FTS complex search`); - return { results: [], totalCount: 0 }; - } - - logger.info(`${FILE_NAME}: Performing complex FTS search with ${parsedQuery.conditions.length} conditions`); - - let db; - let shouldCloseDb = false; - - try { - // Use provided db connection or create a new one - if (providedDb) { - db = providedDb; - } else { - db = await getDbConnection(); - shouldCloseDb = true; - logger.debug(`${FILE_NAME}: Created new database connection for complex FTS search`); - } - - // Build FTS query from conditions - const { ftsQuery, whereClause, params } = buildComplexFtsQuery(parsedQuery); - - logger.debug(`${FILE_NAME}: FTS query: "${ftsQuery}", additional where: ${whereClause ? whereClause : 'none'}`); - logger.debug(`${FILE_NAME}: Query parameters: ${JSON.stringify(params)}`); - - // Build count query - let countQuery; - let countParams; - - if (whereClause) { - countQuery = ` - SELECT COUNT(*) as count - FROM rule_search - WHERE rule_search MATCH ? - AND ${whereClause} - `; - countParams = [ftsQuery, ...params]; - } else { - countQuery = ` - SELECT COUNT(*) as count - FROM rule_search - WHERE rule_search MATCH ? - `; - countParams = [ftsQuery]; - } - - // Get total count - const countResult = await new Promise((resolve, reject) => { - db.get(countQuery, countParams, (err, row) => { - if (err) { - logger.error(`${FILE_NAME}: Complex FTS count query error: ${err.message}`); - reject(err); - } else { - resolve(row || { count: 0 }); - } - }); - }); - - const totalCount = countResult.count; - logger.debug(`${FILE_NAME}: Total matching rules for complex FTS query: ${totalCount}`); - - // Build results query with pagination - let searchQuery; - let searchParams; - - if (whereClause) { - searchQuery = ` - SELECT rule_id, title - FROM rule_search - WHERE rule_search MATCH ? - AND ${whereClause} - ORDER BY rank - LIMIT ? OFFSET ? - `; - searchParams = [ftsQuery, ...params, limit, offset]; - } else { - searchQuery = ` - SELECT rule_id, title - FROM rule_search - WHERE rule_search MATCH ? - ORDER BY rank - LIMIT ? OFFSET ? - `; - searchParams = [ftsQuery, limit, offset]; - } - - // Get paginated results - const results = await new Promise((resolve, reject) => { - db.all(searchQuery, searchParams, (err, rows) => { - if (err) { - logger.error(`${FILE_NAME}: Complex FTS search query error: ${err.message}`); - reject(err); - } else { - logger.debug(`${FILE_NAME}: Complex FTS search query returned ${rows ? rows.length : 0} results`); - resolve(rows || []); - } - }); - }); - - // Format the results - const formattedResults = results.map(r => ({ - id: r.rule_id, - title: r.title || r.rule_id - })); - - logger.debug(`${FILE_NAME}: Returning ${formattedResults.length} results for complex FTS search`); - - return { - results: formattedResults, - totalCount - }; - } catch (error) { - logger.error(`${FILE_NAME}: Error in complex FTS search operation: ${error.message}`); - logger.debug(`${FILE_NAME}: Complex FTS search error stack: ${error.stack}`); - return { results: [], totalCount: 0 }; - } finally { - // Only close the database if we created it AND we're not in the middle of a transaction - if (db && shouldCloseDb) { - try { - await db.close(); - logger.debug(`${FILE_NAME}: Database connection closed after complex FTS search`); - } catch (closeError) { - logger.error(`${FILE_NAME}: Error closing database after complex FTS search: ${closeError.message}`); - } - } - } -} - -/** - * Build FTS query and WHERE clause from parsed query conditions - * - * @param {Object} parsedQuery - The parsed query object - * @returns {Object} Object with FTS query, additional WHERE clause, and parameters - */ -function buildComplexFtsQuery(parsedQuery) { - const { conditions, operator } = parsedQuery; - - // Separate text search conditions from other conditions - const textConditions = []; - const nonTextConditions = []; - - for (const condition of conditions) { - switch (condition.field) { - case 'title': - case 'description': - case 'author': - case 'tags': - case 'keyword': - // These can be handled by FTS directly - textConditions.push(condition); - break; - default: - // These need additional WHERE clauses - nonTextConditions.push(condition); - break; - } - } - - // Build FTS MATCH query - let ftsQueryParts = []; - - for (const condition of textConditions) { - let fieldPrefix = ''; - - // Add field-specific prefix if available - if (condition.field !== 'keyword') { - fieldPrefix = `${condition.field}:`; - } - - // Add wildcard for partial matching if not already present - let value = condition.value.trim(); - if (!value.endsWith('*')) { - value = `${value}*`; - } - - ftsQueryParts.push(`${fieldPrefix}${value}`); - } - - // If no text conditions, use a match-all query - const ftsQuery = ftsQueryParts.length > 0 - ? ftsQueryParts.join(operator === 'AND' ? ' AND ' : ' OR ') - : '*'; - - // Build additional WHERE clauses for non-text conditions - let whereClauseParts = []; - const params = []; - - for (const condition of nonTextConditions) { - switch (condition.field) { - case 'date': - const dateOperator = condition.operator === 'after' ? '>' : - condition.operator === 'before' ? '<' : '='; - whereClauseParts.push(`date ${dateOperator} date(?)`); - params.push(condition.value); - break; - - case 'level': - whereClauseParts.push(`level = ?`); - params.push(condition.value); - break; - - case 'logsource': - whereClauseParts.push(`logsource LIKE ?`); - params.push(`%${condition.subfield}%${condition.value}%`); - break; - - case 'id': - whereClauseParts.push(`rule_id = ?`); - params.push(condition.value); - break; - } - } - - // Combine WHERE clauses - const whereClause = whereClauseParts.length > 0 - ? whereClauseParts.join(operator === 'AND' ? ' AND ' : ' OR ') - : ''; - - return { ftsQuery, whereClause, params }; -} - -/** - * Build the SQL query for complex search based on parsed conditions - * - * @param {Object} parsedQuery - The parsed query object - * @param {number} limit - Results limit - * @param {number} offset - Results offset - * @returns {Object} Object with SQL query, count query, and parameters - */ -function buildComplexSqlQuery(parsedQuery, limit, offset) { - const { conditions, operator } = parsedQuery; - const params = []; - - // Start building the primary table selection - let sqlSelectPart = ` - SELECT DISTINCT r.id as rule_id, - (SELECT param_value FROM rule_parameters WHERE rule_id = r.id AND param_name = 'title' LIMIT 1) as title - FROM sigma_rules r - `; - - // Build WHERE clause based on conditions - let whereClauses = []; - let joinIdx = 0; - - for (const condition of conditions) { - let whereClause = ''; - - switch (condition.field) { - case 'title': - joinIdx++; - whereClause = `EXISTS ( - SELECT 1 FROM rule_parameters p${joinIdx} - WHERE p${joinIdx}.rule_id = r.id - AND p${joinIdx}.param_name = 'title' - AND INSTR(LOWER(p${joinIdx}.param_value), LOWER(?)) > 0 - )`; - params.push(condition.value); - break; - - case 'description': - joinIdx++; - whereClause = `EXISTS ( - SELECT 1 FROM rule_parameters p${joinIdx} - WHERE p${joinIdx}.rule_id = r.id - AND p${joinIdx}.param_name = 'description' - AND INSTR(LOWER(p${joinIdx}.param_value), LOWER(?)) > 0 - )`; - params.push(condition.value); - break; - - case 'logsource': - joinIdx++; - whereClause = `EXISTS ( - SELECT 1 FROM rule_parameters p${joinIdx} - WHERE p${joinIdx}.rule_id = r.id - AND p${joinIdx}.param_name = 'logsource' - AND INSTR(LOWER(p${joinIdx}.param_value), LOWER('"${condition.subfield}":"${condition.value}"')) > 0 - )`; - break; - - case 'tags': - joinIdx++; - whereClause = `EXISTS ( - SELECT 1 FROM rule_parameters p${joinIdx} - WHERE p${joinIdx}.rule_id = r.id - AND p${joinIdx}.param_name = 'tags' - AND INSTR(LOWER(p${joinIdx}.param_value), LOWER(?)) > 0 - )`; - params.push(condition.value); - break; - - case 'date': - joinIdx++; - const dateOperator = condition.operator === 'after' ? '>' : - condition.operator === 'before' ? '<' : '='; - whereClause = `r.date ${dateOperator} date(?)`; - params.push(condition.value); - break; - - case 'author': - joinIdx++; - whereClause = `EXISTS ( - SELECT 1 FROM rule_parameters p${joinIdx} - WHERE p${joinIdx}.rule_id = r.id - AND p${joinIdx}.param_name = 'author' - AND INSTR(LOWER(p${joinIdx}.param_value), LOWER(?)) > 0 - )`; - params.push(condition.value); - break; - - case 'level': - joinIdx++; - whereClause = `EXISTS ( - SELECT 1 FROM rule_parameters p${joinIdx} - WHERE p${joinIdx}.rule_id = r.id - AND p${joinIdx}.param_name = 'level' - AND LOWER(p${joinIdx}.param_value) = LOWER(?) - )`; - params.push(condition.value); - break; - - case 'id': - whereClause = `LOWER(r.id) = LOWER(?)`; - params.push(condition.value); - break; - - case 'keyword': - default: - // Default to searching in title - joinIdx++; - whereClause = `EXISTS ( - SELECT 1 FROM rule_parameters p${joinIdx} - WHERE p${joinIdx}.rule_id = r.id - AND p${joinIdx}.param_name = 'title' - AND INSTR(LOWER(p${joinIdx}.param_value), LOWER(?)) > 0 - )`; - params.push(condition.value); - break; - } - - if (whereClause) { - whereClauses.push(whereClause); - } - } - - // Combine the WHERE clauses with the appropriate operator - let whereStatement = ''; - if (whereClauses.length > 0) { - const combiner = operator === 'AND' ? ' AND ' : ' OR '; - whereStatement = `WHERE ${whereClauses.join(combiner)}`; - } - - // Complete queries - const sqlQuery = ` - ${sqlSelectPart} - ${whereStatement} - ORDER BY rule_id - LIMIT ? OFFSET ? - `; - - const sqlCountQuery = ` - SELECT COUNT(DISTINCT r.id) as count - FROM sigma_rules r - ${whereStatement} - `; - - // Add pagination parameters - params.push(limit); - params.push(offset); - - return { sqlQuery, sqlCountQuery, params }; -} - -/** - * Debug function to retrieve detailed information about a rule's content - * Useful for diagnosing issues with rule retrieval and content parsing - * - * @param {string} ruleId - The ID of the rule to debug - * @returns {Promise} Object containing debug information or null on error - */ -async function debugRuleContent(ruleId) { - if (!ruleId) { - logger.warn(`${FILE_NAME}: Cannot debug rule: Missing rule ID`); - return null; - } - - let db; - try { - db = await getDbConnection(); - - const absolutePath = path.resolve(DB_PATH); - logger.debug(`${FILE_NAME}: Debug function connecting to DB at path: ${absolutePath}`); - - // Get raw rule record - const rule = await db.get('SELECT id, file_path, length(content) as content_length, typeof(content) as content_type FROM sigma_rules WHERE id = ?', [ruleId]); - - if (!rule) { - logger.warn(`${FILE_NAME}: Rule with ID ${ruleId} not found during debug operation`); - return { error: 'Rule not found', ruleId }; - } - - // Return just the rule information without the undefined variables - return { - rule, - ruleId - }; - } catch (error) { - logger.error(`${FILE_NAME}: Debug error for rule ${ruleId}: ${error.message}`); - logger.debug(`${FILE_NAME}: Debug error stack: ${error.stack}`); - return { - error: error.message, - stack: error.stack, - ruleId - }; - } finally { - if (db) { - try { - await db.close(); - logger.debug(`${FILE_NAME}: Database connection closed after debug operation`); - } catch (closeError) { - logger.warn(`${FILE_NAME}: Error closing database after debug: ${closeError.message}`); - } - } - } -} - -/** - * Get the raw YAML content of a Sigma rule - * Retrieves the content field from the database which should contain YAML - * - * @param {string} ruleId - The ID of the rule - * @returns {Promise} Object with success flag and content or error message - */ -async function getRuleYamlContent(ruleId) { - if (!ruleId) { - logger.warn(`${FILE_NAME}: Cannot get YAML content: Missing rule ID`); - return { success: false, message: 'Missing rule ID' }; - } - - let db; - try { - logger.info(`${FILE_NAME}: Fetching YAML content for rule: ${ruleId}`); - logger.debug(`${FILE_NAME}: Rule ID type: ${typeof ruleId}, length: ${ruleId.length}`); - - db = await getDbConnection(); - logger.debug(`${FILE_NAME}: Connected to database for YAML retrieval`); - - // Debug query before execution - const debugResult = await db.get('SELECT id, typeof(content) as content_type, length(content) as content_length FROM sigma_rules WHERE id = ?', [ruleId]); - logger.debug(`${FILE_NAME}: Debug query result: ${JSON.stringify(debugResult || 'not found')}`); - - if (!debugResult) { - logger.warn(`${FILE_NAME}: Rule with ID ${ruleId} not found in debug query`); - return { success: false, message: 'Rule not found' }; - } - - // Get actual content - const rule = await new Promise((resolve, reject) => { - db.get('SELECT content FROM sigma_rules WHERE id = ?', [ruleId], (err, row) => { - if (err) { - logger.error(`${FILE_NAME}: Content query error: ${err.message}`); - reject(err); - } else { - resolve(row || null); - } - }); - }); - logger.debug(`${FILE_NAME}: Content query result for ${ruleId}: ${rule ? 'Found' : 'Not found'}`); - - if (!rule) { - logger.warn(`${FILE_NAME}: Rule with ID ${ruleId} not found in content query`); - return { success: false, message: 'Rule not found' }; - } - - if (!rule.content) { - logger.warn(`${FILE_NAME}: Rule with ID ${ruleId} found but has no content`); - return { success: false, message: 'Rule found but content is empty' }; - } - - logger.debug(`${FILE_NAME}: Content retrieved successfully for ${ruleId}, type: ${typeof rule.content}, length: ${rule.content.length}`); - - return { success: true, content: rule.content }; - } catch (error) { - logger.error(`${FILE_NAME}: Error retrieving YAML content for ${ruleId}: ${error.message}`); - logger.debug(`${FILE_NAME}: YAML retrieval error stack: ${error.stack}`); - return { success: false, message: `Error retrieving YAML: ${error.message}` }; - } finally { - if (db) { - try { - await db.close(); - logger.debug(`${FILE_NAME}: Database connection closed after YAML retrieval`); - } catch (closeError) { - logger.warn(`${FILE_NAME}: Error closing database after YAML retrieval: ${closeError.message}`); - } - } - } -} - -/** - * Get statistics about Sigma rules in the database - * Collects counts, categories, and other aggregate information - * - * @returns {Promise} Object with various statistics about the rules - */ -async function getStatsFromDatabase() { - let db; - try { - db = await getDbConnection(); - logger.debug(`${FILE_NAME}: Connected to database for statistics`); - - // Get total rule count - const totalRulesResult = await new Promise((resolve, reject) => { - db.get('SELECT COUNT(*) as count FROM sigma_rules', (err, row) => { - if (err) reject(err); - else resolve(row); - }); - }); - const totalRules = totalRulesResult.count; - - // Get last update time - const lastUpdateResult = await new Promise((resolve, reject) => { - db.get('SELECT MAX(date) as last_update FROM sigma_rules', (err, row) => { - if (err) reject(err); - else resolve(row); - }); - }); - const lastUpdate = lastUpdateResult.last_update; - - // Get rules by log source count (Windows, Linux, macOS) - const windowsRulesResult = await new Promise((resolve, reject) => { - db.get(` - SELECT COUNT(DISTINCT rule_id) as count - FROM rule_parameters - WHERE param_name = 'logsource' AND - param_value LIKE '%"product":"windows"%'`, - (err, row) => { - if (err) reject(err); - else resolve(row); - }); - }); - const windowsRules = windowsRulesResult.count || 0; - - const linuxRulesResult = await new Promise((resolve, reject) => { - db.get(` - SELECT COUNT(DISTINCT rule_id) as count - FROM rule_parameters - WHERE param_name = 'logsource' AND - param_value LIKE '%"product":"linux"%'`, - (err, row) => { - if (err) reject(err); - else resolve(row); - }); - }); - const linuxRules = linuxRulesResult.count || 0; - - const macosRulesResult = await new Promise((resolve, reject) => { - db.get(` - SELECT COUNT(DISTINCT rule_id) as count - FROM rule_parameters - WHERE param_name = 'logsource' AND - param_value LIKE '%"product":"macos"%'`, - (err, row) => { - if (err) reject(err); - else resolve(row); - }); - }); - const macosRules = macosRulesResult.count || 0; - - // Get rules by severity level - const severityStats = await new Promise((resolve, reject) => { - db.all(` - SELECT param_value AS level, COUNT(DISTINCT rule_id) as count - FROM rule_parameters - WHERE param_name = 'level' - GROUP BY param_value - ORDER BY - CASE - WHEN param_value = 'critical' THEN 1 - WHEN param_value = 'high' THEN 2 - WHEN param_value = 'medium' THEN 3 - WHEN param_value = 'low' THEN 4 - WHEN param_value = 'informational' THEN 5 - ELSE 6 - END`, - (err, rows) => { - if (err) reject(err); - else resolve(rows); - }); - }); - - // Get top 5 rule authors - const topAuthors = await new Promise((resolve, reject) => { - db.all(` - SELECT param_value AS author, COUNT(*) as count - FROM rule_parameters - WHERE param_name = 'author' - GROUP BY param_value - ORDER BY count DESC - LIMIT 5`, - (err, rows) => { - if (err) reject(err); - else resolve(rows); - }); - }); - - // Get empty content count (rules with missing YAML) - const emptyContentResult = await new Promise((resolve, reject) => { - db.get(` - SELECT COUNT(*) as count - FROM sigma_rules - WHERE content IS NULL OR content = ''`, - (err, row) => { - if (err) reject(err); - else resolve(row); - }); - }); - const emptyContentCount = emptyContentResult.count; - - // Get MITRE ATT&CK tactics statistics - const mitreStats = await new Promise((resolve, reject) => { - db.all(` - SELECT param_value AS tag, COUNT(DISTINCT rule_id) as count - FROM rule_parameters - WHERE param_name = 'tags' AND param_value LIKE 'attack.%' - GROUP BY param_value - ORDER BY count DESC - LIMIT 10`, - (err, rows) => { - if (err) reject(err); - else resolve(rows); - }); - }); - - // Format MITRE tactics for display - const formattedMitreTactics = mitreStats.map(item => { - const tactic = item.tag.substring(7); // Remove 'attack.' prefix - return { - tactic: tactic, - count: item.count - }; - }); - - // Compile all statistics - const stats = { - totalRules, - lastUpdate, - operatingSystems: { - windows: windowsRules, - linux: linuxRules, - macos: macosRules, - other: totalRules - (windowsRules + linuxRules + macosRules) - }, - severityLevels: severityStats.map(s => ({ level: s.level, count: s.count })), - topAuthors: topAuthors.map(a => ({ name: a.author, count: a.count })), - databaseHealth: { - emptyContentCount, - contentPercentage: totalRules > 0 ? Math.round(((totalRules - emptyContentCount) / totalRules) * 100) : 0 - }, - mitreTactics: formattedMitreTactics - }; - - return { - success: true, - stats - }; - } catch (error) { - logger.error(`${FILE_NAME}: Error retrieving statistics: ${error.message}`); - logger.debug(`${FILE_NAME}: Error stack: ${error.stack}`); - return { - success: false, - message: `Error retrieving statistics: ${error.message}` - }; - } finally { - if (db) { - try { - await new Promise((resolve) => db.close(() => resolve())); - logger.debug(`${FILE_NAME}: Database connection closed after statistics retrieval`); - } catch (closeError) { - logger.warn(`${FILE_NAME}: Error closing database: ${closeError.message}`); - } - } - } -} - -module.exports = { - getAllRuleIds, - findRuleById, - searchRules, - searchRulesFTS, - searchRulesComplex, - searchRulesComplexFTS, - debugRuleContent, - getRuleYamlContent, - getStatsFromDatabase, - checkFtsAvailable -}; \ No newline at end of file