260 lines
7.6 KiB
JavaScript
260 lines
7.6 KiB
JavaScript
const fs = require('fs');
|
|
const path = require('path');
|
|
const { performance } = require('perf_hooks');
|
|
const {
|
|
config,
|
|
createClient,
|
|
ensureDir,
|
|
readCsv,
|
|
resultsDir,
|
|
summarizeRows,
|
|
writeCsv,
|
|
writeJson
|
|
} = require('./benchmark-lib');
|
|
|
|
const query02Dir = path.join(resultsDir, 'query02');
|
|
const query02Sql = fs
|
|
.readFileSync(path.join(__dirname, '..', 'queries', 'query_02.sql'), 'utf8')
|
|
.trim()
|
|
.replace(/;\s*$/, '');
|
|
|
|
const pageFields = [
|
|
'source_dataset_key',
|
|
'source_dataset_id',
|
|
'source_dataset_name',
|
|
'categoria_linea',
|
|
'categoria_canonica',
|
|
'categoria_slug',
|
|
'expediente',
|
|
'registro_sanitario',
|
|
'producto',
|
|
'titular',
|
|
'estado_registro',
|
|
'fecha_expedicion',
|
|
'fecha_vencimiento',
|
|
'modalidad',
|
|
'grupo',
|
|
'marca',
|
|
'principio_activo',
|
|
'forma_farmaceutica',
|
|
'presentacion_comercial',
|
|
'atc',
|
|
'rol_nombre',
|
|
'rol_tipo',
|
|
'ciudad_titular',
|
|
'pais_titular',
|
|
'fabricante',
|
|
'importador',
|
|
'vigencia'
|
|
];
|
|
|
|
const pageSelectSql = `
|
|
SELECT
|
|
source_dataset_key,
|
|
source_uid,
|
|
${pageFields.filter((field) => field !== 'source_dataset_key').join(',\n ')}
|
|
FROM invima_api_catalog
|
|
ORDER BY
|
|
fecha_vencimiento DESC NULLS LAST,
|
|
producto ASC NULLS LAST,
|
|
source_dataset_key ASC,
|
|
source_uid ASC
|
|
LIMIT $1 OFFSET $2
|
|
`;
|
|
|
|
const countSql = 'SELECT COUNT(*)::bigint AS total FROM invima_api_catalog';
|
|
|
|
const explainSql = (sql) =>
|
|
`EXPLAIN (ANALYZE, BUFFERS, VERBOSE, SETTINGS, WAL, FORMAT JSON) ${sql.replace(/;\s*$/, '')}`;
|
|
|
|
const rowId = (row) => `${row.source_dataset_key}|${row.source_uid}`;
|
|
|
|
const normalizeValue = (value) => {
|
|
if (value instanceof Date) return value.toISOString();
|
|
if (value === undefined) return null;
|
|
return value;
|
|
};
|
|
|
|
const publicShape = (row) =>
|
|
Object.fromEntries(pageFields.map((field) => [field, normalizeValue(row[field])]));
|
|
|
|
const orderKey = (row) =>
|
|
`${normalizeValue(row.fecha_vencimiento) || ''}|${row.producto || ''}|${row.source_dataset_key}|${row.source_uid}`;
|
|
|
|
const duplicates = (items) => {
|
|
const seen = new Set();
|
|
const dupes = new Set();
|
|
for (const item of items) {
|
|
if (seen.has(item)) dupes.add(item);
|
|
seen.add(item);
|
|
}
|
|
return Array.from(dupes);
|
|
};
|
|
|
|
const diffPage = (beforePage, afterPage) => {
|
|
const beforeIds = beforePage.rows.map(rowId);
|
|
const afterIds = afterPage.rows.map(rowId);
|
|
const beforeOrderKeys = beforePage.rows.map(orderKey);
|
|
const afterOrderKeys = afterPage.rows.map(orderKey);
|
|
const beforePublic = beforePage.rows.map(publicShape);
|
|
const afterPublic = afterPage.rows.map(publicShape);
|
|
const beforeSet = new Set(beforeIds);
|
|
const afterSet = new Set(afterIds);
|
|
const missing = beforeIds.filter((id) => !afterSet.has(id));
|
|
const additional = afterIds.filter((id) => !beforeSet.has(id));
|
|
const orderChanged =
|
|
beforeIds.length !== afterIds.length || beforeIds.some((id, index) => id !== afterIds[index]);
|
|
const orderKeysChanged =
|
|
beforeOrderKeys.length !== afterOrderKeys.length ||
|
|
beforeOrderKeys.some((key, index) => key !== afterOrderKeys[index]);
|
|
const structureChanged =
|
|
JSON.stringify(beforePublic.map((row) => Object.keys(row))) !==
|
|
JSON.stringify(afterPublic.map((row) => Object.keys(row)));
|
|
const publicValuesChanged = JSON.stringify(beforePublic) !== JSON.stringify(afterPublic);
|
|
|
|
return {
|
|
offset: beforePage.offset,
|
|
limit: beforePage.limit,
|
|
rowCountBefore: beforePage.rows.length,
|
|
rowCountAfter: afterPage.rows.length,
|
|
missingIds: missing,
|
|
additionalIds: additional,
|
|
duplicateIdsBefore: duplicates(beforeIds),
|
|
duplicateIdsAfter: duplicates(afterIds),
|
|
orderChanged,
|
|
orderKeysChanged,
|
|
idOrderChangedWithinTies: orderChanged && !orderKeysChanged,
|
|
structureChanged,
|
|
publicValuesChanged,
|
|
valid:
|
|
beforePage.rows.length === afterPage.rows.length &&
|
|
missing.length === 0 &&
|
|
additional.length === 0 &&
|
|
duplicates(afterIds).length === 0 &&
|
|
!orderKeysChanged &&
|
|
!structureChanged &&
|
|
!publicValuesChanged
|
|
};
|
|
};
|
|
|
|
const capturePageSet = async (client, offsets = [0, 25, 250000, 1000000], limit = 25) => {
|
|
const countResult = await client.query(countSql);
|
|
const total = Number(countResult.rows[0]?.total || 0);
|
|
const pages = [];
|
|
|
|
for (const offset of offsets) {
|
|
const result = await client.query(pageSelectSql, [limit, offset]);
|
|
pages.push({
|
|
limit,
|
|
offset,
|
|
ids: result.rows.map(rowId),
|
|
orderKeys: result.rows.map(orderKey),
|
|
fields: pageFields,
|
|
rows: result.rows
|
|
});
|
|
}
|
|
|
|
return {
|
|
capturedAt: new Date().toISOString(),
|
|
total,
|
|
pages
|
|
};
|
|
};
|
|
|
|
const compareCaptures = (before, after) => {
|
|
const pageDiffs = before.pages.map((beforePage) => {
|
|
const afterPage = after.pages.find((page) => page.offset === beforePage.offset);
|
|
if (!afterPage) {
|
|
return {
|
|
offset: beforePage.offset,
|
|
valid: false,
|
|
missingAfterPage: true
|
|
};
|
|
}
|
|
return diffPage(beforePage, afterPage);
|
|
});
|
|
|
|
return {
|
|
totalBefore: before.total,
|
|
totalAfter: after.total,
|
|
totalMatches: before.total === after.total,
|
|
pages: pageDiffs,
|
|
valid: before.total === after.total && pageDiffs.every((page) => page.valid)
|
|
};
|
|
};
|
|
|
|
const flattenPlans = (plan) => {
|
|
const nodes = [];
|
|
const visit = (node) => {
|
|
if (!node) return;
|
|
nodes.push(node);
|
|
for (const child of node.Plans || []) visit(child);
|
|
};
|
|
visit(plan);
|
|
return nodes;
|
|
};
|
|
|
|
const explainSummary = (doc) => {
|
|
const plan = doc.Plan || {};
|
|
const nodes = flattenPlans(plan);
|
|
const sum = (field) => nodes.reduce((acc, item) => acc + Number(item[field] || 0), 0);
|
|
const nodeTypes = Array.from(new Set(nodes.map((node) => node['Node Type']))).join(', ');
|
|
return {
|
|
planning_ms: Number(doc['Planning Time'] || 0),
|
|
execution_ms: Number(doc['Execution Time'] || 0),
|
|
node_types: nodeTypes,
|
|
seq_scans: nodes.filter((node) => node['Node Type'] === 'Seq Scan').length,
|
|
index_scans: nodes.filter((node) => node['Node Type'] === 'Index Scan').length,
|
|
index_only_scans: nodes.filter((node) => node['Node Type'] === 'Index Only Scan').length,
|
|
bitmap_heap_scans: nodes.filter((node) => node['Node Type'] === 'Bitmap Heap Scan').length,
|
|
sorts: nodes.filter((node) => node['Node Type'] === 'Sort').length,
|
|
incremental_sorts: nodes.filter((node) => node['Node Type'] === 'Incremental Sort').length,
|
|
gathers: nodes.filter((node) => node['Node Type'] === 'Gather').length,
|
|
gather_merges: nodes.filter((node) => node['Node Type'] === 'Gather Merge').length,
|
|
estimated_rows: Number(plan['Plan Rows'] || 0),
|
|
actual_rows: Number(plan['Actual Rows'] || 0),
|
|
estimated_width: Number(plan['Plan Width'] || 0),
|
|
rows_removed_by_filter: sum('Rows Removed by Filter'),
|
|
shared_hit_blocks: sum('Shared Hit Blocks'),
|
|
shared_read_blocks: sum('Shared Read Blocks'),
|
|
temp_read_blocks: sum('Temp Read Blocks'),
|
|
temp_written_blocks: sum('Temp Written Blocks'),
|
|
wal_records: sum('WAL Records'),
|
|
sort_methods: nodes
|
|
.filter((node) => node['Sort Method'])
|
|
.map((node) => `${node['Sort Method']} ${node['Sort Space Used'] || ''}${node['Sort Space Type'] || ''}`)
|
|
};
|
|
};
|
|
|
|
const timedQuery = async (client, sql, params = []) => {
|
|
const started = performance.now();
|
|
const result = await client.query(sql, params);
|
|
return {
|
|
result,
|
|
elapsed_ms: performance.now() - started
|
|
};
|
|
};
|
|
|
|
const query02ResultFile = (name) => path.join(query02Dir, name);
|
|
|
|
module.exports = {
|
|
capturePageSet,
|
|
compareCaptures,
|
|
config,
|
|
countSql,
|
|
createClient,
|
|
ensureDir,
|
|
explainSql,
|
|
explainSummary,
|
|
pageFields,
|
|
pageSelectSql,
|
|
query02Dir,
|
|
query02ResultFile,
|
|
query02Sql,
|
|
readCsv,
|
|
summarizeRows,
|
|
timedQuery,
|
|
writeCsv,
|
|
writeJson
|
|
};
|