refactor sigma_db_queries into multiple files

src/services/sigma/sigma_backend_converter.js

@@ -11,7 +11,7 @@ const { execSync } = require('child_process');
 const logger = require('../../utils/logger');
 const { SIGMA_CLI_PATH, SIGMA_CLI_CONFIG } = require('../../config/appConfig');
 const { convertSigmaRule } = require('./sigma_converter_service');
-const { getRuleYamlContent } = require('../../sigma_db/sigma_db_queries');
+const { getRuleYamlContent } = require('../../sigma_db/queries');
 
 const { getFileName } = require('../../utils/file_utils');
 const FILE_NAME = getFileName(__filename);

src/services/sigma/sigma_converter_service.js

@@ -4,7 +4,7 @@
 //
 const logger = require('../../utils/logger');
 const yaml = require('js-yaml');
-const { findRuleById } = require('../../sigma_db/sigma_db_queries');
+const { findRuleById } = require('../../sigma_db/queries');
 
 const { getFileName } = require('../../utils/file_utils');
 const FILE_NAME = getFileName(__filename);

src/services/sigma/sigma_details_service.js

@@ -5,7 +5,7 @@
  */
 const logger = require('../../utils/logger');
 const { convertSigmaRule, extractDetectionCondition } = require('./sigma_converter_service');
-const { debugRuleContent, getRuleYamlContent } = require('../../sigma_db/sigma_db_queries');
+const { debugRuleContent, getRuleYamlContent } = require('../../sigma_db/queries');
 
 const { getFileName } = require('../../utils/file_utils');
 const FILE_NAME = getFileName(__filename);
@@ -147,4 +147,4 @@ async function getSigmaRuleYaml(ruleId) {
 module.exports = {
   explainSigmaRule,
   getSigmaRuleYaml
-};
+};

src/services/sigma/sigma_search_service.js

@@ -6,7 +6,7 @@
  * Supports pagination for large result sets.
  */
 
-const { searchRules, searchRulesComplex } = require('../../sigma_db/sigma_db_queries');
+const { searchRules, searchRulesComplex } = require('../../sigma_db/queries');
 const { parseComplexQuery } = require('../../lang/query_parser');
 const logger = require('../../utils/logger');
 const { convertSigmaRule } = require('./sigma_converter_service');

src/services/sigma/sigma_stats_service.js

@@ -5,7 +5,7 @@
  * Provides aggregated statistical information about the rule database
  */
 const logger = require('../../utils/logger');
-const { getStatsFromDatabase } = require('../../sigma_db/sigma_db_queries');
+const { getStatsFromDatabase } = require('../../sigma_db/queries');
 
 const { getFileName } = require('../../utils/file_utils');
 const FILE_NAME = getFileName(__filename);

src/sigma_db/queries/complex-search.js

@@ -0,0 +1,259 @@
+/**
+ * complex-search.js
+ * Functions for complex searching of Sigma rules
+ */
+
+const { getDbConnection } = require('../sigma_db_connection');
+const logger = require('../../utils/logger');
+const { getFileName } = require('../../utils/file_utils');
+const { checkFtsAvailable } = require('./fts-search');
+const { buildComplexSqlQuery, buildComplexFtsQuery } = require('./query-builders');
+const FILE_NAME = getFileName(__filename);
+
+/**
+ * Search for Sigma rules using complex query conditions
+ * Supports filtering by multiple attributes like title, logsource, tags, etc.
+ * 
+ * @param {Object} parsedQuery - The parsed query object containing conditions and operator
+ * @param {number} limit - Maximum number of results to return
+ * @param {number} offset - Number of results to skip (for pagination)
+ * @returns {Promise<Object>} Object with results array and total count
+ */
+async function searchRulesComplex(parsedQuery, limit = 10, offset = 0) {
+  if (!parsedQuery || !parsedQuery.valid) {
+    logger.warn(`${FILE_NAME}: Invalid query object provided`);
+    return { results: [], totalCount: 0 };
+  }
+
+  logger.info(`${FILE_NAME}: Performing complex search with ${parsedQuery.conditions.length} conditions (limit: ${limit}, offset: ${offset})`);
+
+  let db;
+  // Declare this at function scope so it's available in the finally block
+  let usingFts = false;
+
+  try {
+    db = await getDbConnection();
+    logger.debug(`${FILE_NAME}: Database connection established for complex search`);
+
+    // Check if FTS5 is available
+    const ftsAvailable = await checkFtsAvailable(db);
+
+    if (ftsAvailable) {
+      logger.debug(`${FILE_NAME}: Using FTS5 for complex search`);
+      // Set flag that we're using FTS
+      usingFts = true;
+      // Pass db connection to searchRulesComplexFTS and let that function manage it
+      const results = await searchRulesComplexFTS(parsedQuery, limit, offset, db);
+      return results;
+    }
+
+    logger.debug(`${FILE_NAME}: FTS5 not available, using legacy complex search method`);
+
+    // Build the SQL query based on the conditions
+    const { sqlQuery, sqlCountQuery, params } = buildComplexSqlQuery(parsedQuery, limit, offset);
+
+    logger.debug(`${FILE_NAME}: Executing complex search SQL: ${sqlQuery}`);
+    logger.debug(`${FILE_NAME}: Query parameters: ${JSON.stringify(params)}`);
+
+    // First get the total count of matching results
+    const countResult = await new Promise((resolve, reject) => {
+      db.get(sqlCountQuery, params.slice(0, params.length - 2), (err, row) => {
+        if (err) {
+          logger.error(`${FILE_NAME}: Complex search count query error: ${err.message}`);
+          reject(err);
+        } else {
+          resolve(row || { count: 0 });
+        }
+      });
+    });
+
+    const totalCount = countResult.count;
+    logger.debug(`${FILE_NAME}: Total matching rules for complex query: ${totalCount}`);
+
+    // Now get the actual results with pagination
+    const results = await new Promise((resolve, reject) => {
+      db.all(sqlQuery, params, (err, rows) => {
+        if (err) {
+          logger.error(`${FILE_NAME}: Complex search query error: ${err.message}`);
+          reject(err);
+        } else {
+          logger.debug(`${FILE_NAME}: Complex search query returned ${rows ? rows.length : 0} results`);
+          resolve(rows || []);
+        }
+      });
+    });
+
+    // Format the results
+    const formattedResults = results.map(r => ({
+      id: r.rule_id,
+      title: r.title || r.rule_id
+    }));
+
+    logger.debug(`${FILE_NAME}: Returning ${formattedResults.length} results for complex search`);
+
+    return {
+      results: formattedResults,
+      totalCount
+    };
+  } catch (error) {
+    logger.error(`${FILE_NAME}: Error in complex search operation: ${error.message}`);
+    logger.debug(`${FILE_NAME}: Complex search error stack: ${error.stack}`);
+    return { results: [], totalCount: 0 };
+  } finally {
+    // IMPORTANT: Only close the db connection if we're not using FTS
+    // When using FTS, let searchRulesComplexFTS manage the connection
+    if (db && !usingFts) {
+      try {
+        await new Promise((resolve) => db.close(() => resolve()));
+        logger.debug(`${FILE_NAME}: Database connection closed after complex search operation`);
+      } catch (closeError) {
+        logger.error(`${FILE_NAME}: Error closing database after complex search: ${closeError.message}`);
+      }
+    }
+  }
+}
+
+/**
+ * Search for Sigma rules using complex query conditions with FTS5
+ * Uses the FTS5 virtual table for faster text searching
+ * 
+ * @param {Object} parsedQuery - The parsed query object 
+ * @param {number} limit - Maximum number of results to return
+ * @param {number} offset - Number of results to skip (for pagination)
+ * @param {Object} providedDb - Database connection (optional, will create one if not provided)
+ * @returns {Promise<Object>} Object with results array and total count
+ */
+async function searchRulesComplexFTS(parsedQuery, limit = 10, offset = 0, providedDb = null) {
+  if (!parsedQuery || !parsedQuery.valid) {
+    logger.warn(`${FILE_NAME}: Invalid query object provided for FTS complex search`);
+    return { results: [], totalCount: 0 };
+  }
+
+  logger.info(`${FILE_NAME}: Performing complex FTS search with ${parsedQuery.conditions.length} conditions`);
+
+  let db;
+  let shouldCloseDb = false;
+
+  try {
+    // Use provided db connection or create a new one
+    if (providedDb) {
+      db = providedDb;
+    } else {
+      db = await getDbConnection();
+      shouldCloseDb = true;
+      logger.debug(`${FILE_NAME}: Created new database connection for complex FTS search`);
+    }
+
+    // Build FTS query from conditions
+    const { ftsQuery, whereClause, params } = buildComplexFtsQuery(parsedQuery);
+
+    logger.debug(`${FILE_NAME}: FTS query: "${ftsQuery}", additional where: ${whereClause ? whereClause : 'none'}`);
+    logger.debug(`${FILE_NAME}: Query parameters: ${JSON.stringify(params)}`);
+
+    // Build count query
+    let countQuery;
+    let countParams;
+
+    if (whereClause) {
+      countQuery = `
+        SELECT COUNT(*) as count
+        FROM rule_search
+        WHERE rule_search MATCH ?
+        AND ${whereClause}
+      `;
+      countParams = [ftsQuery, ...params];
+    } else {
+      countQuery = `
+        SELECT COUNT(*) as count
+        FROM rule_search
+        WHERE rule_search MATCH ?
+      `;
+      countParams = [ftsQuery];
+    }
+
+    // Get total count
+    const countResult = await new Promise((resolve, reject) => {
+      db.get(countQuery, countParams, (err, row) => {
+        if (err) {
+          logger.error(`${FILE_NAME}: Complex FTS count query error: ${err.message}`);
+          reject(err);
+        } else {
+          resolve(row || { count: 0 });
+        }
+      });
+    });
+
+    const totalCount = countResult.count;
+    logger.debug(`${FILE_NAME}: Total matching rules for complex FTS query: ${totalCount}`);
+
+    // Build results query with pagination
+    let searchQuery;
+    let searchParams;
+
+    if (whereClause) {
+      searchQuery = `
+        SELECT rule_id, title
+        FROM rule_search
+        WHERE rule_search MATCH ?
+        AND ${whereClause}
+        ORDER BY rank
+        LIMIT ? OFFSET ?
+      `;
+      searchParams = [ftsQuery, ...params, limit, offset];
+    } else {
+      searchQuery = `
+        SELECT rule_id, title
+        FROM rule_search
+        WHERE rule_search MATCH ?
+        ORDER BY rank
+        LIMIT ? OFFSET ?
+      `;
+      searchParams = [ftsQuery, limit, offset];
+    }
+
+    // Get paginated results
+    const results = await new Promise((resolve, reject) => {
+      db.all(searchQuery, searchParams, (err, rows) => {
+        if (err) {
+          logger.error(`${FILE_NAME}: Complex FTS search query error: ${err.message}`);
+          reject(err);
+        } else {
+          logger.debug(`${FILE_NAME}: Complex FTS search query returned ${rows ? rows.length : 0} results`);
+          resolve(rows || []);
+        }
+      });
+    });
+
+    // Format the results
+    const formattedResults = results.map(r => ({
+      id: r.rule_id,
+      title: r.title || r.rule_id
+    }));
+
+    logger.debug(`${FILE_NAME}: Returning ${formattedResults.length} results for complex FTS search`);
+
+    return {
+      results: formattedResults,
+      totalCount
+    };
+  } catch (error) {
+    logger.error(`${FILE_NAME}: Error in complex FTS search operation: ${error.message}`);
+    logger.debug(`${FILE_NAME}: Complex FTS search error stack: ${error.stack}`);
+    return { results: [], totalCount: 0 };
+  } finally {
+    // Only close the database if we created it AND we're not in the middle of a transaction
+    if (db && shouldCloseDb) {
+      try {
+        await db.close();
+        logger.debug(`${FILE_NAME}: Database connection closed after complex FTS search`);
+      } catch (closeError) {
+        logger.error(`${FILE_NAME}: Error closing database after complex FTS search: ${closeError.message}`);
+      }
+    }
+  }
+}
+
+module.exports = {
+  searchRulesComplex,
+  searchRulesComplexFTS
+};

src/sigma_db/queries/fts-search.js

@@ -0,0 +1,133 @@
+/**
+ * fts-search.js
+ * Functions for Full Text Search (FTS) of Sigma rules
+ */
+
+const { getDbConnection } = require('../sigma_db_connection');
+const logger = require('../../utils/logger');
+const { getFileName } = require('../../utils/file_utils');
+const FILE_NAME = getFileName(__filename);
+
+/**
+ * Check if FTS5 virtual table is available
+ * 
+ * @param {Object} db - Database connection
+ * @returns {Promise<boolean>} Whether FTS5 is available
+ */
+async function checkFtsAvailable(db) {
+  try {
+    const result = await new Promise((resolve, reject) => {
+      db.get("SELECT name FROM sqlite_master WHERE type='table' AND name='rule_search'", (err, row) => {
+        if (err) {
+          logger.error(`${FILE_NAME}: Error checking for FTS5 table: ${err.message}`);
+          reject(err);
+        } else {
+          resolve(row !== undefined);
+        }
+      });
+    });
+
+    logger.debug(`${FILE_NAME}: FTS5 table availability check: ${result ? 'Available' : 'Not available'}`);
+    return result;
+  } catch (error) {
+    logger.error(`${FILE_NAME}: Error checking FTS availability: ${error.message}`);
+    return false;
+  }
+}
+
+/**
+ * Search for Sigma rules using FTS5
+ * Performs a full-text search and returns matching rules with pagination
+ * 
+ * @param {string} keyword - The keyword to search for
+ * @param {number} limit - Maximum number of results to return (default: 10)
+ * @param {number} offset - Number of results to skip (for pagination, default: 0)
+ * @returns {Promise<Object>} Object with results array and total count
+ */
+async function searchRulesFTS(keyword, limit = 10, offset = 0) {
+  if (!keyword) {
+    logger.warn(`${FILE_NAME}: Empty search keyword provided for FTS search`);
+    return { results: [], totalCount: 0 };
+  }
+
+  // Prepare FTS query - add * for prefix matching if not already present
+  let ftsQuery = keyword.trim();
+  if (!ftsQuery.endsWith('*')) {
+    ftsQuery = `${ftsQuery}*`;
+  }
+
+  logger.info(`${FILE_NAME}: Performing FTS search with query: "${ftsQuery}" (limit: ${limit}, offset: ${offset})`);
+
+  let db;
+  try {
+    db = await getDbConnection();
+    logger.debug(`${FILE_NAME}: Database connection established for FTS search`);
+
+    // First get the total count of matching rules
+    const countQuery = `
+      SELECT COUNT(*) as count
+      FROM rule_search
+      WHERE rule_search MATCH ?
+    `;
+
+    const countResult = await new Promise((resolve, reject) => {
+      db.get(countQuery, [ftsQuery], (err, row) => {
+        if (err) {
+          logger.error(`${FILE_NAME}: FTS count query error: ${err.message}`);
+          reject(err);
+        } else {
+          resolve(row || { count: 0 });
+        }
+      });
+    });
+
+    const totalCount = countResult.count;
+    logger.debug(`${FILE_NAME}: Total matching rules for FTS query "${ftsQuery}": ${totalCount}`);
+
+    // Now get the actual results with pagination
+    const searchQuery = `
+      SELECT rule_id, title
+      FROM rule_search
+      WHERE rule_search MATCH ?
+      ORDER BY rank
+      LIMIT ? OFFSET ?
+    `;
+
+    const results = await new Promise((resolve, reject) => {
+      db.all(searchQuery, [ftsQuery, limit, offset], (err, rows) => {
+        if (err) {
+          logger.error(`${FILE_NAME}: FTS search query error: ${err.message}`);
+          reject(err);
+        } else {
+          logger.debug(`${FILE_NAME}: FTS search query returned ${rows ? rows.length : 0} results`);
+          resolve(rows || []);
+        }
+      });
+    });
+
+    logger.debug(`${FILE_NAME}: FTS search results page for query "${ftsQuery}": ${results.length} matches (page ${Math.floor(offset / limit) + 1})`);
+
+    return {
+      results: results.map(r => ({ id: r.rule_id, title: r.title || r.rule_id })),
+      totalCount
+    };
+  } catch (error) {
+    logger.error(`${FILE_NAME}: Error in FTS search operation: ${error.message}`);
+    logger.debug(`${FILE_NAME}: FTS search error stack: ${error.stack}`);
+    return { results: [], totalCount: 0 };
+  } finally {
+    if (db) {
+      try {
+        await db.close();
+        logger.debug(`${FILE_NAME}: Database connection closed after FTS search operation`);
+      } catch (closeError) {
+        logger.error(`${FILE_NAME}: Error closing database connection after FTS search: ${closeError.message}`);
+      }
+    }
+  }
+}
+
+module.exports = {
+  checkFtsAvailable,
+  searchRulesFTS
+};

src/sigma_db/queries/index.js

@@ -0,0 +1,34 @@
+/**
+ * index.js
+ * Central module for accessing all Sigma database query functions
+ */
+
+// Import functions from individual modules
+const ruleRetrieval = require('./rule-retrieval');
+const simpleSearch = require('./simple-search');
+const ftsSearch = require('./fts-search');
+const complexSearch = require('./complex-search');
+const statsDebug = require('./stats-debug');
+
+// Export all functions
+module.exports = {
+  // Rule retrieval functions
+  getAllRuleIds: ruleRetrieval.getAllRuleIds,
+  findRuleById: ruleRetrieval.findRuleById,
+  
+  // Search functions
+  searchRules: simpleSearch.searchRules,
+  
+  // FTS search functions
+  searchRulesFTS: ftsSearch.searchRulesFTS,
+  checkFtsAvailable: ftsSearch.checkFtsAvailable,
+  
+  // Complex search functions
+  searchRulesComplex: complexSearch.searchRulesComplex,
+  searchRulesComplexFTS: complexSearch.searchRulesComplexFTS,
+  
+  // Stats and debug functions
+  debugRuleContent: statsDebug.debugRuleContent,
+  getRuleYamlContent: statsDebug.getRuleYamlContent,
+  getStatsFromDatabase: statsDebug.getStatsFromDatabase
+};

src/sigma_db/queries/query-builders.js

@@ -0,0 +1,258 @@
+/**
+ * query-builders.js
+ * Helper functions for building SQL queries
+ */
+
+const logger = require('../../utils/logger');
+const { getFileName } = require('../../utils/file_utils');
+const FILE_NAME = getFileName(__filename);
+
+/**
+ * Build the SQL query for complex search based on parsed conditions
+ * 
+ * @param {Object} parsedQuery - The parsed query object
+ * @param {number} limit - Results limit
+ * @param {number} offset - Results offset
+ * @returns {Object} Object with SQL query, count query, and parameters
+ */
+function buildComplexSqlQuery(parsedQuery, limit, offset) {
+  const { conditions, operator } = parsedQuery;
+  const params = [];
+
+  // Start building the primary table selection
+  let sqlSelectPart = `
+    SELECT DISTINCT r.id as rule_id, 
+      (SELECT param_value FROM rule_parameters WHERE rule_id = r.id AND param_name = 'title' LIMIT 1) as title
+    FROM sigma_rules r
+  `;
+
+  // Build WHERE clause based on conditions
+  let whereClauses = [];
+  let joinIdx = 0;
+
+  for (const condition of conditions) {
+    let whereClause = '';
+
+    switch (condition.field) {
+      case 'title':
+        joinIdx++;
+        whereClause = `EXISTS (
+          SELECT 1 FROM rule_parameters p${joinIdx}
+          WHERE p${joinIdx}.rule_id = r.id
+          AND p${joinIdx}.param_name = 'title'
+          AND INSTR(LOWER(p${joinIdx}.param_value), LOWER(?)) > 0
+        )`;
+        params.push(condition.value);
+        break;
+
+      case 'description':
+        joinIdx++;
+        whereClause = `EXISTS (
+          SELECT 1 FROM rule_parameters p${joinIdx}
+          WHERE p${joinIdx}.rule_id = r.id
+          AND p${joinIdx}.param_name = 'description'
+          AND INSTR(LOWER(p${joinIdx}.param_value), LOWER(?)) > 0
+        )`;
+        params.push(condition.value);
+        break;
+
+      case 'logsource':
+        joinIdx++;
+        whereClause = `EXISTS (
+          SELECT 1 FROM rule_parameters p${joinIdx}
+          WHERE p${joinIdx}.rule_id = r.id
+          AND p${joinIdx}.param_name = 'logsource'
+          AND INSTR(LOWER(p${joinIdx}.param_value), LOWER('"${condition.subfield}":"${condition.value}"')) > 0
+        )`;
+        break;
+
+      case 'tags':
+        joinIdx++;
+        whereClause = `EXISTS (
+          SELECT 1 FROM rule_parameters p${joinIdx}
+          WHERE p${joinIdx}.rule_id = r.id
+          AND p${joinIdx}.param_name = 'tags'
+          AND INSTR(LOWER(p${joinIdx}.param_value), LOWER(?)) > 0
+        )`;
+        params.push(condition.value);
+        break;
+
+      case 'date':
+        joinIdx++;
+        const dateOperator = condition.operator === 'after' ? '>' :
+          condition.operator === 'before' ? '<' : '=';
+        whereClause = `r.date ${dateOperator} date(?)`;
+        params.push(condition.value);
+        break;
+
+      case 'author':
+        joinIdx++;
+        whereClause = `EXISTS (
+          SELECT 1 FROM rule_parameters p${joinIdx}
+          WHERE p${joinIdx}.rule_id = r.id
+          AND p${joinIdx}.param_name = 'author'
+          AND INSTR(LOWER(p${joinIdx}.param_value), LOWER(?)) > 0
+        )`;
+        params.push(condition.value);
+        break;
+
+      case 'level':
+        joinIdx++;
+        whereClause = `EXISTS (
+          SELECT 1 FROM rule_parameters p${joinIdx}
+          WHERE p${joinIdx}.rule_id = r.id
+          AND p${joinIdx}.param_name = 'level'
+          AND LOWER(p${joinIdx}.param_value) = LOWER(?)
+        )`;
+        params.push(condition.value);
+        break;
+
+      case 'id':
+        whereClause = `LOWER(r.id) = LOWER(?)`;
+        params.push(condition.value);
+        break;
+
+      case 'keyword':
+      default:
+        // Default to searching in title
+        joinIdx++;
+        whereClause = `EXISTS (
+          SELECT 1 FROM rule_parameters p${joinIdx}
+          WHERE p${joinIdx}.rule_id = r.id
+          AND p${joinIdx}.param_name = 'title'
+          AND INSTR(LOWER(p${joinIdx}.param_value), LOWER(?)) > 0
+        )`;
+        params.push(condition.value);
+        break;
+    }
+
+    if (whereClause) {
+      whereClauses.push(whereClause);
+    }
+  }
+
+  // Combine the WHERE clauses with the appropriate operator
+  let whereStatement = '';
+  if (whereClauses.length > 0) {
+    const combiner = operator === 'AND' ? ' AND ' : ' OR ';
+    whereStatement = `WHERE ${whereClauses.join(combiner)}`;
+  }
+
+  // Complete queries
+  const sqlQuery = `
+    ${sqlSelectPart}
+    ${whereStatement}
+    ORDER BY rule_id
+    LIMIT ? OFFSET ?
+  `;
+
+  const sqlCountQuery = `
+    SELECT COUNT(DISTINCT r.id) as count
+    FROM sigma_rules r
+    ${whereStatement}
+  `;
+
+  // Add pagination parameters
+  params.push(limit);
+  params.push(offset);
+
+  return { sqlQuery, sqlCountQuery, params };
+}
+
+/**
+ * Build FTS query and WHERE clause from parsed query conditions
+ * 
+ * @param {Object} parsedQuery - The parsed query object
+ * @returns {Object} Object with FTS query, additional WHERE clause, and parameters
+ */
+function buildComplexFtsQuery(parsedQuery) {
+  const { conditions, operator } = parsedQuery;
+
+  // Separate text search conditions from other conditions
+  const textConditions = [];
+  const nonTextConditions = [];
+
+  for (const condition of conditions) {
+    switch (condition.field) {
+      case 'title':
+      case 'description':
+      case 'author':
+      case 'tags':
+      case 'keyword':
+        // These can be handled by FTS directly
+        textConditions.push(condition);
+        break;
+      default:
+        // These need additional WHERE clauses
+        nonTextConditions.push(condition);
+        break;
+    }
+  }
+
+  // Build FTS MATCH query
+  let ftsQueryParts = [];
+
+  for (const condition of textConditions) {
+    let fieldPrefix = '';
+
+    // Add field-specific prefix if available
+    if (condition.field !== 'keyword') {
+      fieldPrefix = `${condition.field}:`;
+    }
+
+    // Add wildcard for partial matching if not already present
+    let value = condition.value.trim();
+    if (!value.endsWith('*')) {
+      value = `${value}*`;
+    }
+
+    ftsQueryParts.push(`${fieldPrefix}${value}`);
+  }
+
+  // If no text conditions, use a match-all query
+  const ftsQuery = ftsQueryParts.length > 0
+    ? ftsQueryParts.join(operator === 'AND' ? ' AND ' : ' OR ')
+    : '*';
+
+  // Build additional WHERE clauses for non-text conditions
+  let whereClauseParts = [];
+  const params = [];
+
+  for (const condition of nonTextConditions) {
+    switch (condition.field) {
+      case 'date':
+        const dateOperator = condition.operator === 'after' ? '>' :
+          condition.operator === 'before' ? '<' : '=';
+        whereClauseParts.push(`date ${dateOperator} date(?)`);
+        params.push(condition.value);
+        break;
+
+      case 'level':
+        whereClauseParts.push(`level = ?`);
+        params.push(condition.value);
+        break;
+
+      case 'logsource':
+        whereClauseParts.push(`logsource LIKE ?`);
+        params.push(`%${condition.subfield}%${condition.value}%`);
+        break;
+
+      case 'id':
+        whereClauseParts.push(`rule_id = ?`);
+        params.push(condition.value);
+        break;
+    }
+  }
+
+  // Combine WHERE clauses
+  const whereClause = whereClauseParts.length > 0
+    ? whereClauseParts.join(operator === 'AND' ? ' AND ' : ' OR ')
+    : '';
+
+  return { ftsQuery, whereClause, params };
+}
+
+module.exports = {
+  buildComplexSqlQuery,
+  buildComplexFtsQuery
+};

src/sigma_db/queries/rule-retrieval.js

@@ -0,0 +1,164 @@
+/**
+ * rule-retrieval.js
+ * Functions for retrieving Sigma rules and rule IDs
+ */
+
+const { getDbConnection } = require('../sigma_db_connection');
+const logger = require('../../utils/logger');
+const { getFileName } = require('../../utils/file_utils');
+const FILE_NAME = getFileName(__filename);
+
+/**
+ * Get a list of all rule IDs in the database
+ * Useful for bulk operations and database integrity checks
+ * 
+ * @returns {Promise<Array>} Array of rule IDs or empty array on error
+ */
+async function getAllRuleIds() {
+  let db;
+  try {
+    logger.info(`${FILE_NAME}: Retrieving all rule IDs from database`);
+
+    db = await getDbConnection();
+    logger.debug(`${FILE_NAME}: Connected to database for retrieving all rule IDs`);
+
+    const result = await new Promise((resolve, reject) => {
+      db.all('SELECT id FROM sigma_rules ORDER BY id', [], (err, rows) => {
+        if (err) {
+          logger.error(`${FILE_NAME}: Error fetching all rule IDs: ${err.message}`);
+          reject(err);
+        } else {
+          resolve(rows || []);
+        }
+      });
+    });
+
+    logger.debug(`${FILE_NAME}: Retrieved ${result.length} rule IDs from database`);
+    return result.map(row => row.id);
+  } catch (error) {
+    logger.error(`${FILE_NAME}: Error retrieving all rule IDs: ${error.message}`);
+    logger.debug(`${FILE_NAME}: Error stack: ${error.stack}`);
+    return [];
+  } finally {
+    if (db) {
+      try {
+        await db.close();
+        logger.debug(`${FILE_NAME}: Database connection closed after retrieving all rule IDs`);
+      } catch (closeError) {
+        logger.warn(`${FILE_NAME}: Error closing database: ${closeError.message}`);
+      }
+    }
+  }
+}
+
+/**
+ * Find a Sigma rule by its ID
+ * Retrieves rule data and associated parameters from the database
+ * 
+ * @param {string} ruleId - The ID of the rule to find
+ * @returns {Promise<Object|null>} The rule object or null if not found
+ */
+async function findRuleById(ruleId) {
+  if (!ruleId) {
+    logger.warn(`${FILE_NAME}: Cannot find rule: Missing rule ID`);
+    return null;
+  }
+
+  let db;
+  try {
+    db = await getDbConnection();
+    logger.debug(`${FILE_NAME}: Connected to database for rule lookup: ${ruleId}`);
+
+    // Get the base rule using promisified method
+    const rule = await db.getAsync('SELECT * FROM sigma_rules WHERE id = ?', [ruleId]);
+    if (!rule) {
+      logger.warn(`${FILE_NAME}: Rule with ID ${ruleId} not found in database`);
+      return null;
+    }
+
+    logger.debug(`${FILE_NAME}: Found base rule with ID ${ruleId}, content length: ${rule.content ? rule.content.length : 0}`);
+
+    // Get parameters using promisified method
+    const paramsAsync = await db.allAsync('SELECT param_name, param_value, param_type FROM rule_parameters WHERE rule_id = ?', [ruleId]);
+    logger.debug(`${FILE_NAME}: Params query returned ${paramsAsync ? paramsAsync.length : 0} results via allAsync`);
+
+    // Check if content is missing
+    if (!rule.content) {
+      logger.warn(`${FILE_NAME}: Rule with ID ${ruleId} exists but has no content`);
+      rule.content_missing = true;
+    }
+
+    // Get all parameters for this rule with case-insensitive matching
+    try {
+      const params = await new Promise((resolve, reject) => {
+        db.all(
+          'SELECT param_name, param_value, param_type FROM rule_parameters WHERE LOWER(rule_id) = LOWER(?)',
+          [ruleId],
+          (err, rows) => {
+            if (err) reject(err);
+            else resolve(rows);
+          }
+        );
+      });
+
+      logger.debug(`${FILE_NAME}: Retrieved ${params ? params.length : 0} parameters for rule ${ruleId}`);
+
+      // Validate params is an array
+      if (params && Array.isArray(params)) {
+        // Attach parameters to the rule object
+        rule.parameters = {};
+
+        for (const param of params) {
+          if (param && param.param_name) {
+            // Convert value based on type
+            let value = param.param_value;
+
+            if (param.param_type === 'object' || param.param_type === 'array') {
+              try {
+                value = JSON.parse(param.param_value);
+              } catch (parseError) {
+                logger.warn(`${FILE_NAME}: Failed to parse JSON for parameter ${param.param_name}: ${parseError.message}`);
+              }
+            } else if (param.param_type === 'boolean') {
+              value = param.param_value === 'true';
+            } else if (param.param_type === 'number') {
+              value = Number(param.param_value);
+            }
+
+            rule.parameters[param.param_name] = value;
+          }
+        }
+
+        logger.debug(`${FILE_NAME}: Successfully processed ${Object.keys(rule.parameters).length} parameters for rule ${ruleId}`);
+      } else {
+        logger.warn(`${FILE_NAME}: Parameters for rule ${ruleId} not available or not iterable`);
+        rule.parameters = {};
+      }
+    } catch (paramError) {
+      logger.error(`${FILE_NAME}: Error fetching parameters for rule ${ruleId}: ${paramError.message}`);
+      logger.debug(`${FILE_NAME}: Parameter error stack: ${paramError.stack}`);
+      rule.parameters = {};
+    }
+
+    return rule;
+  } catch (error) {
+    logger.error(`${FILE_NAME}: Error finding rule ${ruleId}: ${error.message}`);
+    logger.debug(`${FILE_NAME}: Error stack: ${error.stack}`);
+    return null;
+  } finally {
+    // Close the database connection if it was opened
+    if (db && typeof db.close === 'function') {
+      try {
+        await db.close();
+        logger.debug(`${FILE_NAME}: Database connection closed after rule lookup`);
+      } catch (closeError) {
+        logger.warn(`${FILE_NAME}: Error closing database connection: ${closeError.message}`);
+      }
+    }
+  }
+}
+
+module.exports = {
+  getAllRuleIds,
+  findRuleById
+};

src/sigma_db/queries/simple-search.js

@@ -0,0 +1,118 @@
+/**
+ * simple-search.js
+ * Functions for basic search of Sigma rules
+ */
+
+const { getDbConnection } = require('../sigma_db_connection');
+const logger = require('../../utils/logger');
+const { getFileName } = require('../../utils/file_utils');
+const FILE_NAME = getFileName(__filename);
+
+// Import FTS functions - need to use relative path for proper circular dependency handling
+const { checkFtsAvailable, searchRulesFTS } = require('./fts-search');
+
+/**
+ * Search for Sigma rules by keyword in rule titles
+ * Performs a case-insensitive search and returns matching rules with pagination
+ * 
+ * @param {string} keyword - The keyword to search for
+ * @param {number} limit - Maximum number of results to return (default: 10)
+ * @param {number} offset - Number of results to skip (for pagination, default: 0)
+ * @returns {Promise<Object>} Object with results array and total count
+ */
+async function searchRules(keyword, limit = 10, offset = 0) {
+  if (!keyword) {
+    logger.warn(`${FILE_NAME}: Empty search keyword provided`);
+    return { results: [], totalCount: 0 };
+  }
+
+  // Sanitize keyword to prevent SQL injection
+  const sanitizedKeyword = keyword.replace(/'/g, "''");
+  logger.info(`${FILE_NAME}: Searching for rules with keyword in title: ${sanitizedKeyword} (limit: ${limit}, offset: ${offset})`);
+
+  let db;
+  try {
+    // Make sure we properly await the DB connection
+    db = await getDbConnection();
+    logger.debug(`${FILE_NAME}: Database connection established for search`);
+
+    // Use FTS5 for faster searching if available
+    const ftsAvailable = await checkFtsAvailable(db);
+
+    if (ftsAvailable) {
+      logger.debug(`${FILE_NAME}: Using FTS5 for keyword search`);
+      return searchRulesFTS(keyword, limit, offset);
+    }
+
+    // If FTS5 is not available, use the legacy search method
+    logger.debug(`${FILE_NAME}: FTS5 not available, using legacy search method`);
+
+    // First get the total count of matching rules (for pagination info)
+    const countQuery = `
+      SELECT COUNT(*) as count
+      FROM rule_parameters
+      WHERE param_name = 'title'
+      AND INSTR(LOWER(param_value), LOWER(?)) > 0
+    `;
+
+    const countResult = await new Promise((resolve, reject) => {
+      db.get(countQuery, [sanitizedKeyword], (err, row) => {
+        if (err) {
+          logger.error(`${FILE_NAME}: Count query error: ${err.message}`);
+          reject(err);
+        } else {
+          resolve(row || { count: 0 });
+        }
+      });
+    });
+
+    const totalCount = countResult.count;
+    logger.debug(`${FILE_NAME}: Total matching rules for "${sanitizedKeyword}": ${totalCount}`);
+
+    // Use parameterized query instead of string interpolation for better security
+    const instrQuery = `
+      SELECT rule_id, param_value AS title
+      FROM rule_parameters
+      WHERE param_name = 'title'
+      AND INSTR(LOWER(param_value), LOWER(?)) > 0
+      LIMIT ? OFFSET ?
+    `;
+
+    const results = await new Promise((resolve, reject) => {
+      db.all(instrQuery, [sanitizedKeyword, limit, offset], (err, rows) => {
+        if (err) {
+          logger.error(`${FILE_NAME}: Search query error: ${err.message}`);
+          reject(err);
+        } else {
+          logger.debug(`${FILE_NAME}: Search query returned ${rows ? rows.length : 0} results`);
+          resolve(rows || []);
+        }
+      });
+    });
+
+    logger.debug(`${FILE_NAME}: Search results page for keyword "${sanitizedKeyword}": ${results.length} matches (page ${Math.floor(offset / limit) + 1})`);
+
+    return {
+      results: results.map(r => ({ id: r.rule_id, title: r.title })),
+      totalCount
+    };
+  } catch (error) {
+    logger.error(`${FILE_NAME}: Error in search operation: ${error.message}`);
+    logger.debug(`${FILE_NAME}: Search error stack: ${error.stack}`);
+    return { results: [], totalCount: 0 };
+  } finally {
+    // Make sure we properly close the connection
+    if (db) {
+      try {
+        await new Promise((resolve) => db.close(() => resolve()));
+        logger.debug(`${FILE_NAME}: Database connection closed after search operation`);
+      } catch (closeError) {
+        logger.error(`${FILE_NAME}: Error closing database connection after search: ${closeError.message}`);
+      }
+    }
+  }
+}
+
+module.exports = {
+  searchRules
+};

src/sigma_db/queries/stats-debug.js

@@ -0,0 +1,328 @@
+/**
+ * stats-debug.js
+ * Functions for database statistics and debugging
+ */
+
+const { getDbConnection } = require('../sigma_db_connection');
+const logger = require('../../utils/logger');
+const { DB_PATH } = require('../../config/appConfig');
+const path = require('path');
+const { getFileName } = require('../../utils/file_utils');
+const FILE_NAME = getFileName(__filename);
+
+/**
+ * Debug function to retrieve detailed information about a rule's content
+ * Useful for diagnosing issues with rule retrieval and content parsing
+ * 
+ * @param {string} ruleId - The ID of the rule to debug
+ * @returns {Promise<Object|null>} Object containing debug information or null on error
+ */
+async function debugRuleContent(ruleId) {
+  if (!ruleId) {
+    logger.warn(`${FILE_NAME}: Cannot debug rule: Missing rule ID`);
+    return null;
+  }
+
+  let db;
+  try {
+    db = await getDbConnection();
+
+    const absolutePath = path.resolve(DB_PATH);
+    logger.debug(`${FILE_NAME}: Debug function connecting to DB at path: ${absolutePath}`);
+
+    // Get raw rule record
+    const rule = await db.get('SELECT id, file_path, length(content) as content_length, typeof(content) as content_type FROM sigma_rules WHERE id = ?', [ruleId]);
+
+    if (!rule) {
+      logger.warn(`${FILE_NAME}: Rule with ID ${ruleId} not found during debug operation`);
+      return { error: 'Rule not found', ruleId };
+    }
+
+    // Return just the rule information without the undefined variables
+    return {
+      rule,
+      ruleId
+    };
+  } catch (error) {
+    logger.error(`${FILE_NAME}: Debug error for rule ${ruleId}: ${error.message}`);
+    logger.debug(`${FILE_NAME}: Debug error stack: ${error.stack}`);
+    return {
+      error: error.message,
+      stack: error.stack,
+      ruleId
+    };
+  } finally {
+    if (db) {
+      try {
+        await db.close();
+        logger.debug(`${FILE_NAME}: Database connection closed after debug operation`);
+      } catch (closeError) {
+        logger.warn(`${FILE_NAME}: Error closing database after debug: ${closeError.message}`);
+      }
+    }
+  }
+}
+
+/**
+ * Get the raw YAML content of a Sigma rule
+ * Retrieves the content field from the database which should contain YAML
+ * 
+ * @param {string} ruleId - The ID of the rule
+ * @returns {Promise<Object>} Object with success flag and content or error message
+ */
+async function getRuleYamlContent(ruleId) {
+  if (!ruleId) {
+    logger.warn(`${FILE_NAME}: Cannot get YAML content: Missing rule ID`);
+    return { success: false, message: 'Missing rule ID' };
+  }
+
+  let db;
+  try {
+    logger.info(`${FILE_NAME}: Fetching YAML content for rule: ${ruleId}`);
+    logger.debug(`${FILE_NAME}: Rule ID type: ${typeof ruleId}, length: ${ruleId.length}`);
+
+    db = await getDbConnection();
+    logger.debug(`${FILE_NAME}: Connected to database for YAML retrieval`);
+
+    // Debug query before execution
+    const debugResult = await db.get('SELECT id, typeof(content) as content_type, length(content) as content_length FROM sigma_rules WHERE id = ?', [ruleId]);
+    logger.debug(`${FILE_NAME}: Debug query result: ${JSON.stringify(debugResult || 'not found')}`);
+
+    if (!debugResult) {
+      logger.warn(`${FILE_NAME}: Rule with ID ${ruleId} not found in debug query`);
+      return { success: false, message: 'Rule not found' };
+    }
+
+    // Get actual content
+    const rule = await new Promise((resolve, reject) => {
+      db.get('SELECT content FROM sigma_rules WHERE id = ?', [ruleId], (err, row) => {
+        if (err) {
+          logger.error(`${FILE_NAME}: Content query error: ${err.message}`);
+          reject(err);
+        } else {
+          resolve(row || null);
+        }
+      });
+    });
+    logger.debug(`${FILE_NAME}: Content query result for ${ruleId}: ${rule ? 'Found' : 'Not found'}`);
+
+    if (!rule) {
+      logger.warn(`${FILE_NAME}: Rule with ID ${ruleId} not found in content query`);
+      return { success: false, message: 'Rule not found' };
+    }
+
+    if (!rule.content) {
+      logger.warn(`${FILE_NAME}: Rule with ID ${ruleId} found but has no content`);
+      return { success: false, message: 'Rule found but content is empty' };
+    }
+
+    logger.debug(`${FILE_NAME}: Content retrieved successfully for ${ruleId}, type: ${typeof rule.content}, length: ${rule.content.length}`);
+
+    return { success: true, content: rule.content };
+  } catch (error) {
+    logger.error(`${FILE_NAME}: Error retrieving YAML content for ${ruleId}: ${error.message}`);
+    logger.debug(`${FILE_NAME}: YAML retrieval error stack: ${error.stack}`);
+    return { success: false, message: `Error retrieving YAML: ${error.message}` };
+  } finally {
+    if (db) {
+      try {
+        await db.close();
+        logger.debug(`${FILE_NAME}: Database connection closed after YAML retrieval`);
+      } catch (closeError) {
+        logger.warn(`${FILE_NAME}: Error closing database after YAML retrieval: ${closeError.message}`);
+      }
+    }
+  }
+}
+
+/**
+ * Get statistics about Sigma rules in the database
+ * Collects counts, categories, and other aggregate information
+ * 
+ * @returns {Promise<Object>} Object with various statistics about the rules
+ */
+async function getStatsFromDatabase() {
+  let db;
+  try {
+    db = await getDbConnection();
+    logger.debug(`${FILE_NAME}: Connected to database for statistics`);
+
+    // Get total rule count
+    const totalRulesResult = await new Promise((resolve, reject) => {
+      db.get('SELECT COUNT(*) as count FROM sigma_rules', (err, row) => {
+        if (err) reject(err);
+        else resolve(row);
+      });
+    });
+    const totalRules = totalRulesResult.count;
+
+    // Get last update time
+    const lastUpdateResult = await new Promise((resolve, reject) => {
+      db.get('SELECT MAX(date) as last_update FROM sigma_rules', (err, row) => {
+        if (err) reject(err);
+        else resolve(row);
+      });
+    });
+    const lastUpdate = lastUpdateResult.last_update;
+
+    // Get rules by log source count (Windows, Linux, macOS)
+    const windowsRulesResult = await new Promise((resolve, reject) => {
+      db.get(`
+        SELECT COUNT(DISTINCT rule_id) as count 
+        FROM rule_parameters 
+        WHERE param_name = 'logsource' AND 
+        param_value LIKE '%"product":"windows"%'`,
+        (err, row) => {
+          if (err) reject(err);
+          else resolve(row);
+        });
+    });
+    const windowsRules = windowsRulesResult.count || 0;
+
+    const linuxRulesResult = await new Promise((resolve, reject) => {
+      db.get(`
+        SELECT COUNT(DISTINCT rule_id) as count 
+        FROM rule_parameters 
+        WHERE param_name = 'logsource' AND 
+        param_value LIKE '%"product":"linux"%'`,
+        (err, row) => {
+          if (err) reject(err);
+          else resolve(row);
+        });
+    });
+    const linuxRules = linuxRulesResult.count || 0;
+
+    const macosRulesResult = await new Promise((resolve, reject) => {
+      db.get(`
+        SELECT COUNT(DISTINCT rule_id) as count 
+        FROM rule_parameters 
+        WHERE param_name = 'logsource' AND 
+        param_value LIKE '%"product":"macos"%'`,
+        (err, row) => {
+          if (err) reject(err);
+          else resolve(row);
+        });
+    });
+    const macosRules = macosRulesResult.count || 0;
+
+    // Get rules by severity level
+    const severityStats = await new Promise((resolve, reject) => {
+      db.all(`
+        SELECT param_value AS level, COUNT(DISTINCT rule_id) as count 
+        FROM rule_parameters 
+        WHERE param_name = 'level' 
+        GROUP BY param_value 
+        ORDER BY 
+          CASE 
+            WHEN param_value = 'critical' THEN 1 
+            WHEN param_value = 'high' THEN 2 
+            WHEN param_value = 'medium' THEN 3 
+            WHEN param_value = 'low' THEN 4 
+            WHEN param_value = 'informational' THEN 5 
+            ELSE 6 
+          END`,
+        (err, rows) => {
+          if (err) reject(err);
+          else resolve(rows);
+        });
+    });
+
+    // Get top 5 rule authors
+    const topAuthors = await new Promise((resolve, reject) => {
+      db.all(`
+        SELECT param_value AS author, COUNT(*) as count 
+        FROM rule_parameters 
+        WHERE param_name = 'author' 
+        GROUP BY param_value 
+        ORDER BY count DESC 
+        LIMIT 5`,
+        (err, rows) => {
+          if (err) reject(err);
+          else resolve(rows);
+        });
+    });
+
+    // Get empty content count (rules with missing YAML)
+    const emptyContentResult = await new Promise((resolve, reject) => {
+      db.get(`
+        SELECT COUNT(*) as count 
+        FROM sigma_rules 
+        WHERE content IS NULL OR content = ''`,
+        (err, row) => {
+          if (err) reject(err);
+          else resolve(row);
+        });
+    });
+    const emptyContentCount = emptyContentResult.count;
+
+    // Get MITRE ATT&CK tactics statistics
+    const mitreStats = await new Promise((resolve, reject) => {
+      db.all(`
+        SELECT param_value AS tag, COUNT(DISTINCT rule_id) as count 
+        FROM rule_parameters 
+        WHERE param_name = 'tags' AND param_value LIKE 'attack.%' 
+        GROUP BY param_value 
+        ORDER BY count DESC 
+        LIMIT 10`,
+        (err, rows) => {
+          if (err) reject(err);
+          else resolve(rows);
+        });
+    });
+
+    // Format MITRE tactics for display
+    const formattedMitreTactics = mitreStats.map(item => {
+      const tactic = item.tag.substring(7); // Remove 'attack.' prefix
+      return {
+        tactic: tactic,
+        count: item.count
+      };
+    });
+
+    // Compile all statistics
+    const stats = {
+      totalRules,
+      lastUpdate,
+      operatingSystems: {
+        windows: windowsRules,
+        linux: linuxRules,
+        macos: macosRules,
+        other: totalRules - (windowsRules + linuxRules + macosRules)
+      },
+      severityLevels: severityStats.map(s => ({ level: s.level, count: s.count })),
+      topAuthors: topAuthors.map(a => ({ name: a.author, count: a.count })),
+      databaseHealth: {
+        emptyContentCount,
+        contentPercentage: totalRules > 0 ? Math.round(((totalRules - emptyContentCount) / totalRules) * 100) : 0
+      },
+      mitreTactics: formattedMitreTactics
+    };
+
+    return {
+      success: true,
+      stats
+    };
+  } catch (error) {
+    logger.error(`${FILE_NAME}: Error retrieving statistics: ${error.message}`);
+    logger.debug(`${FILE_NAME}: Error stack: ${error.stack}`);
+    return {
+      success: false,
+      message: `Error retrieving statistics: ${error.message}`
+    };
+  } finally {
+    if (db) {
+      try {
+        await new Promise((resolve) => db.close(() => resolve()));
+        logger.debug(`${FILE_NAME}: Database connection closed after statistics retrieval`);
+      } catch (closeError) {
+        logger.warn(`${FILE_NAME}: Error closing database: ${closeError.message}`);
+      }
+    }
+  }
+}
+
+module.exports = {
+  debugRuleContent,
+  getRuleYamlContent,
+  getStatsFromDatabase
+};