semillero-INVIMA/benchmarks/scripts/generate-search-optimization-report.js

const fs = require('fs');
const path = require('path');
const { Client } = require('../../backend/node_modules/pg');
const { getDbConfig, readCsv, resultsDir, benchmarkDir } = require('./benchmark-lib');

const fmt = (value, suffix = '') => {
  const num = Number(value);
  if (!Number.isFinite(num)) return 'No disponible';
  return `${num.toLocaleString('es-CO', { maximumFractionDigits: 4 })}${suffix}`;
};

const readJsonMaybe = (file) => {
  if (!fs.existsSync(file)) return null;
  return JSON.parse(fs.readFileSync(file, 'utf8'));
};

const byQuery = (rows, queryId) => rows.find((row) => row.query_id === queryId);

const improve = (before, after) => {
  const b = Number(before);
  const a = Number(after);
  if (!Number.isFinite(b) || !Number.isFinite(a)) return { absolute: null, percent: null };
  return {
    absolute: b - a,
    percent: b ? ((b - a) / b) * 100 : null
  };
};

const planHighlights = (explain) => {
  const plan = explain?.explain?.Plan;
  if (!plan) return {};
  const nodes = [];
  const visit = (node) => {
    nodes.push(node);
    for (const child of node.Plans || []) visit(child);
  };
  visit(plan);
  return {
    executionMs: explain.explain['Execution Time'],
    planningMs: explain.explain['Planning Time'],
    nodeTypes: Array.from(new Set(nodes.map((node) => node['Node Type']))).join(', '),
    seqScans: nodes.filter((node) => node['Node Type'] === 'Seq Scan').length,
    bitmapScans: nodes.filter((node) => String(node['Node Type']).includes('Bitmap')).length,
    sharedReadBlocks: nodes.reduce((acc, node) => acc + Number(node['Shared Read Blocks'] || 0), 0),
    sharedHitBlocks: nodes.reduce((acc, node) => acc + Number(node['Shared Hit Blocks'] || 0), 0),
    rowsRemoved: nodes.reduce((acc, node) => acc + Number(node['Rows Removed by Filter'] || 0), 0)
  };
};

const indexSizes = async () => {
  const client = new Client(getDbConfig());
  await client.connect();
  try {
    const result = await client.query(`
      SELECT
        c.relname AS name,
        pg_size_pretty(pg_relation_size(c.oid)) AS size,
        pg_relation_size(c.oid) AS bytes,
        i.indisvalid AS valid,
        i.indisready AS ready
      FROM pg_class c
      JOIN pg_index i ON i.indexrelid = c.oid
      JOIN pg_class t ON t.oid = i.indrelid
      WHERE t.relname = 'invima_api_catalog'
        AND c.relname LIKE 'idx_bench_%'
      ORDER BY pg_relation_size(c.oid) DESC, c.relname;
    `);
    return result.rows;
  } finally {
    await client.end();
  }
};

const rowTable = (rows) => {
  if (!rows.length) return 'No hay datos disponibles.';
  return [
    '| Indice | Tamano | Valido | Listo |',
    '|---|---:|---|---|',
    ...rows.map((row) => `| \`${row.name}\` | ${row.size} | ${row.valid} | ${row.ready} |`)
  ].join('\n');
};

const resultLine = (queryId, before, after) => {
  if (!before || !after) return `- ${queryId}: datos incompletos.`;
  const avg = improve(before.latency_avg_ms, after.latency_avg_ms);
  const p95 = improve(before.latency_p95_ms, after.latency_p95_ms);
  return `- ${queryId}: promedio ${fmt(before.latency_avg_ms, ' ms')} -> ${fmt(after.latency_avg_ms, ' ms')} (mejora ${fmt(avg.absolute, ' ms')}, ${fmt(avg.percent, ' %')}); p95 ${fmt(before.latency_p95_ms, ' ms')} -> ${fmt(after.latency_p95_ms, ' ms')} (mejora ${fmt(p95.absolute, ' ms')}, ${fmt(p95.percent, ' %')}).`;
};

const criterion = (queryId, row) => {
  if (!row) return 'No evaluado';
  const avg = Number(row.latency_avg_ms);
  const p95 = Number(row.latency_p95_ms);
  if (queryId === 'query_06') {
    return avg <= 3000 && p95 <= 5000 ? 'Cumple objetivo textual' : 'No cumple objetivo textual';
  }
  return avg <= 5000 && p95 <= 8000 ? 'Cumple objetivo de conteo' : 'No cumple objetivo de conteo';
};

const main = async () => {
  const baseline = readCsv(path.join(resultsDir, 'baseline_summary.csv'));
  const optimized = readCsv(path.join(resultsDir, 'optimized_summary.csv'));
  const validation = readJsonMaybe(path.join(resultsDir, 'validation_search_results.json'));
  const before06 = planHighlights(readJsonMaybe(path.join(resultsDir, 'explain_before', 'query_06.json')));
  const before07 = planHighlights(readJsonMaybe(path.join(resultsDir, 'explain_before', 'query_07.json')));
  const after06 = planHighlights(readJsonMaybe(path.join(resultsDir, 'explain_after', 'query_06.json')));
  const after07 = planHighlights(readJsonMaybe(path.join(resultsDir, 'explain_after', 'query_07.json')));
  const sizes = await indexSizes();

  const b6 = byQuery(baseline, 'query_06');
  const b7 = byQuery(baseline, 'query_07');
  const o6 = byQuery(optimized, 'query_06');
  const o7 = byQuery(optimized, 'query_07');
  const generatedAt = new Date().toISOString();

  const report = `# Informe de optimizacion de busqueda textual

Generado: ${generatedAt}

## 1. Problema encontrado

Las consultas \`query_06\` y \`query_07\` evaluan la busqueda textual de \`marcapasos\` combinando busqueda FTS sobre \`search_text\` con condiciones \`ILIKE '%marcapasos%'\` sobre \`registro_sanitario\`, \`expediente\`, \`producto\`, \`categoria_canonica\` y \`categoria_linea\`. Antes de optimizar, PostgreSQL eligio \`Parallel Seq Scan\`, por lo que recorrio la tabla completa y aplico el filtro despues de leer cientos de miles de bloques.

## 2. Plan SQL anterior

- query_06: ${before06.nodeTypes || 'No disponible'}; Seq Scan=${before06.seqScans ?? 'N/D'}; bloques leidos=${fmt(before06.sharedReadBlocks)}; bloques hit=${fmt(before06.sharedHitBlocks)}; filas removidas=${fmt(before06.rowsRemoved)}; ejecucion=${fmt(before06.executionMs, ' ms')}.
- query_07: ${before07.nodeTypes || 'No disponible'}; Seq Scan=${before07.seqScans ?? 'N/D'}; bloques leidos=${fmt(before07.sharedReadBlocks)}; bloques hit=${fmt(before07.sharedHitBlocks)}; filas removidas=${fmt(before07.rowsRemoved)}; ejecucion=${fmt(before07.executionMs, ' ms')}.

## 3. Cambios implementados

Se mantuvo intacta la semantica de las consultas. La migracion agrega indices GIN trigram de expresion sobre los mismos \`COALESCE(..., '')\` usados por los predicados \`ILIKE\`, elimina el indice de orden de benchmark si estaba invalido y aumenta estadisticas de columnas de busqueda. \`ANALYZE\` se dejo como accion manual porque en esta base local falla por bytes con codificacion invalida en datos existentes.

## 4. Indices nuevos

${rowTable(sizes)}

## 5. Validacion funcional

- Conteo antes: ${validation?.comparison ? fmt(validation.comparison.countBefore) : 'No disponible'}.
- Conteo despues: ${validation?.comparison ? fmt(validation.comparison.countAfter) : 'No disponible'}.
- Faltantes: ${validation?.comparison ? validation.comparison.missingIds.length : 'No disponible'}.
- Adicionales: ${validation?.comparison ? validation.comparison.additionalIds.length : 'No disponible'}.
- Duplicados despues: ${validation?.comparison ? validation.comparison.duplicateIdsAfter.length : 'No disponible'}.
- Cambio de orden: ${validation?.comparison ? validation.comparison.orderChanged : 'No disponible'}.
- Cambio de claves ORDER BY: ${validation?.comparison ? validation.comparison.orderKeysChanged : 'No disponible'}.
- Nota de orden: ${validation?.comparison?.note || 'Sin observaciones'}
- Resultado: ${validation?.comparison?.valid ? 'VALIDO, sin diferencias funcionales' : 'No validado o con diferencias'}.

## 6. Resultados antes y despues

${resultLine('query_06', b6, o6)}
${resultLine('query_07', b7, o7)}

## 7. Plan SQL posterior

- query_06: ${after06.nodeTypes || 'No disponible'}; Seq Scan=${after06.seqScans ?? 'N/D'}; Bitmap=${after06.bitmapScans ?? 'N/D'}; bloques leidos=${fmt(after06.sharedReadBlocks)}; bloques hit=${fmt(after06.sharedHitBlocks)}; ejecucion=${fmt(after06.executionMs, ' ms')}.
- query_07: ${after07.nodeTypes || 'No disponible'}; Seq Scan=${after07.seqScans ?? 'N/D'}; Bitmap=${after07.bitmapScans ?? 'N/D'}; bloques leidos=${fmt(after07.sharedReadBlocks)}; bloques hit=${fmt(after07.sharedHitBlocks)}; ejecucion=${fmt(after07.executionMs, ' ms')}.

## 8. Criterios

- query_06: ${criterion('query_06', o6)}. Objetivo: promedio <= 3 s, p95 <= 5 s, ideal < 1 s.
- query_07: ${criterion('query_07', o7)}. Objetivo: promedio <= 5 s, p95 <= 8 s.

## 9. Riesgos e impacto

Los indices GIN adicionales consumen espacio y pueden aumentar el costo de futuras inserciones o actualizaciones en columnas textuales. No modifican datos regulatorios ni cambian resultados. El conteo exacto sigue evaluando todo el conjunto de coincidencias; si no cumple el objetivo, la siguiente alternativa es reescribir la busqueda como \`UNION\`/deduplicacion exacta o separar el conteo exacto como operacion diferida sin cambiar la ruta que lo obtiene.

## 10. Reversion y repeticion

Revertir:

\`\`\`powershell
npm.cmd run benchmark:rollback
\`\`\`

Repetir pruebas:

\`\`\`powershell
$env:BENCHMARK_QUERY_IDS="query_06,query_07"
$env:BENCHMARK_REPETITIONS="100"
$env:BENCHMARK_WARMUPS="5"
npm.cmd run benchmark:optimized
node benchmarks/scripts/validate-search-results.js after
npm.cmd run benchmark:search:report
\`\`\`

EA, MSE, precision y exhaustividad no calculables por ausencia de un conjunto de referencia validado.
`;

  fs.writeFileSync(path.join(benchmarkDir, 'INFORME_OPTIMIZACION_BUSQUEDA.md'), report, 'utf8');
};

main().catch((error) => {
  console.error(error);
  process.exit(1);
});