mirror of
https://github.com/p-stream/backend.git
synced 2026-01-11 20:10:33 +00:00
Add cron jobs to make timed metrics
/metrics /metrics/monthly /metrics/weekly /metrics/daily
This commit is contained in:
parent
863748730d
commit
b7f8d8899c
12 changed files with 1935 additions and 3786 deletions
3
.gitignore
vendored
3
.gitignore
vendored
|
|
@ -8,3 +8,6 @@ dist
|
||||||
.vscode
|
.vscode
|
||||||
.metrics.json
|
.metrics.json
|
||||||
pnpm-lock.yaml
|
pnpm-lock.yaml
|
||||||
|
.metrics_weekly.json
|
||||||
|
.metrics_monthly.json
|
||||||
|
.metrics_daily.json
|
||||||
|
|
|
||||||
|
|
@ -7,6 +7,15 @@ export default defineNitroConfig({
|
||||||
compatibilityDate: '2025-03-05',
|
compatibilityDate: '2025-03-05',
|
||||||
experimental: {
|
experimental: {
|
||||||
asyncContext: true,
|
asyncContext: true,
|
||||||
|
tasks: true,
|
||||||
|
},
|
||||||
|
scheduledTasks: {
|
||||||
|
// Daily cron jobs (midnight)
|
||||||
|
'0 0 * * *': ['jobs:clear-metrics:daily'],
|
||||||
|
// Weekly cron jobs (Sunday midnight)
|
||||||
|
'0 0 * * 0': ['jobs:clear-metrics:weekly'],
|
||||||
|
// Monthly cron jobs (1st of month at midnight)
|
||||||
|
'0 0 1 * *': ['jobs:clear-metrics:monthly']
|
||||||
},
|
},
|
||||||
runtimeConfig: {
|
runtimeConfig: {
|
||||||
public: {
|
public: {
|
||||||
|
|
|
||||||
5114
pnpm-lock.yaml
5114
pnpm-lock.yaml
File diff suppressed because it is too large
Load diff
33
server/api/jobs/run.post.ts
Normal file
33
server/api/jobs/run.post.ts
Normal file
|
|
@ -0,0 +1,33 @@
|
||||||
|
import { defineEventHandler, getQuery, readBody, createError } from 'h3';
|
||||||
|
import { runTask } from '#imports';
|
||||||
|
|
||||||
|
export default defineEventHandler(async (event) => {
|
||||||
|
// Get job name from query parameters
|
||||||
|
const query = getQuery(event);
|
||||||
|
const jobName = query.job as string;
|
||||||
|
|
||||||
|
if (!jobName) {
|
||||||
|
throw createError({
|
||||||
|
statusCode: 400,
|
||||||
|
statusMessage: "Missing job parameter"
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
// Run the specified task
|
||||||
|
const result = await runTask(jobName, {
|
||||||
|
payload: await readBody(event).catch(() => ({}))
|
||||||
|
});
|
||||||
|
|
||||||
|
return {
|
||||||
|
success: true,
|
||||||
|
job: jobName,
|
||||||
|
result
|
||||||
|
};
|
||||||
|
} catch (error) {
|
||||||
|
throw createError({
|
||||||
|
statusCode: 500,
|
||||||
|
statusMessage: `Failed to run job: ${error.message || 'Unknown error'}`
|
||||||
|
});
|
||||||
|
}
|
||||||
|
});
|
||||||
24
server/routes/metrics/daily.ts
Normal file
24
server/routes/metrics/daily.ts
Normal file
|
|
@ -0,0 +1,24 @@
|
||||||
|
import { getRegistry } from '../../utils/metrics';
|
||||||
|
import { scopedLogger } from '../../utils/logger';
|
||||||
|
|
||||||
|
const log = scopedLogger('metrics-daily-endpoint');
|
||||||
|
|
||||||
|
export default defineEventHandler(async event => {
|
||||||
|
try {
|
||||||
|
// Get the daily registry
|
||||||
|
const dailyRegistry = getRegistry('daily');
|
||||||
|
|
||||||
|
const metrics = await dailyRegistry.metrics();
|
||||||
|
event.node.res.setHeader('Content-Type', dailyRegistry.contentType);
|
||||||
|
return metrics;
|
||||||
|
} catch (error) {
|
||||||
|
log.error('Error in daily metrics endpoint:', {
|
||||||
|
evt: 'metrics_error',
|
||||||
|
error: error instanceof Error ? error.message : String(error),
|
||||||
|
});
|
||||||
|
throw createError({
|
||||||
|
statusCode: 500,
|
||||||
|
message: error instanceof Error ? error.message : 'Failed to collect daily metrics',
|
||||||
|
});
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
@ -1,5 +1,5 @@
|
||||||
import { register } from 'prom-client';
|
import { register } from 'prom-client';
|
||||||
import { setupMetrics } from '../../utils/metrics';
|
import { setupMetrics, initializeAllMetrics } from '../../utils/metrics';
|
||||||
import { scopedLogger } from '../../utils/logger';
|
import { scopedLogger } from '../../utils/logger';
|
||||||
|
|
||||||
const log = scopedLogger('metrics-endpoint');
|
const log = scopedLogger('metrics-endpoint');
|
||||||
|
|
@ -9,7 +9,7 @@ let isInitialized = false;
|
||||||
async function ensureMetricsInitialized() {
|
async function ensureMetricsInitialized() {
|
||||||
if (!isInitialized) {
|
if (!isInitialized) {
|
||||||
log.info('Initializing metrics from endpoint...', { evt: 'init_start' });
|
log.info('Initializing metrics from endpoint...', { evt: 'init_start' });
|
||||||
await setupMetrics();
|
await initializeAllMetrics();
|
||||||
isInitialized = true;
|
isInitialized = true;
|
||||||
log.info('Metrics initialized from endpoint', { evt: 'init_complete' });
|
log.info('Metrics initialized from endpoint', { evt: 'init_complete' });
|
||||||
}
|
}
|
||||||
|
|
@ -18,6 +18,7 @@ async function ensureMetricsInitialized() {
|
||||||
export default defineEventHandler(async event => {
|
export default defineEventHandler(async event => {
|
||||||
try {
|
try {
|
||||||
await ensureMetricsInitialized();
|
await ensureMetricsInitialized();
|
||||||
|
// Use the default registry (all-time metrics)
|
||||||
const metrics = await register.metrics();
|
const metrics = await register.metrics();
|
||||||
event.node.res.setHeader('Content-Type', register.contentType);
|
event.node.res.setHeader('Content-Type', register.contentType);
|
||||||
return metrics;
|
return metrics;
|
||||||
|
|
|
||||||
24
server/routes/metrics/monthly.ts
Normal file
24
server/routes/metrics/monthly.ts
Normal file
|
|
@ -0,0 +1,24 @@
|
||||||
|
import { getRegistry } from '../../utils/metrics';
|
||||||
|
import { scopedLogger } from '../../utils/logger';
|
||||||
|
|
||||||
|
const log = scopedLogger('metrics-monthly-endpoint');
|
||||||
|
|
||||||
|
export default defineEventHandler(async event => {
|
||||||
|
try {
|
||||||
|
// Get the monthly registry
|
||||||
|
const monthlyRegistry = getRegistry('monthly');
|
||||||
|
|
||||||
|
const metrics = await monthlyRegistry.metrics();
|
||||||
|
event.node.res.setHeader('Content-Type', monthlyRegistry.contentType);
|
||||||
|
return metrics;
|
||||||
|
} catch (error) {
|
||||||
|
log.error('Error in monthly metrics endpoint:', {
|
||||||
|
evt: 'metrics_error',
|
||||||
|
error: error instanceof Error ? error.message : String(error),
|
||||||
|
});
|
||||||
|
throw createError({
|
||||||
|
statusCode: 500,
|
||||||
|
message: error instanceof Error ? error.message : 'Failed to collect monthly metrics',
|
||||||
|
});
|
||||||
|
}
|
||||||
|
});
|
||||||
24
server/routes/metrics/weekly.ts
Normal file
24
server/routes/metrics/weekly.ts
Normal file
|
|
@ -0,0 +1,24 @@
|
||||||
|
import { getRegistry } from '../../utils/metrics';
|
||||||
|
import { scopedLogger } from '../../utils/logger';
|
||||||
|
|
||||||
|
const log = scopedLogger('metrics-weekly-endpoint');
|
||||||
|
|
||||||
|
export default defineEventHandler(async event => {
|
||||||
|
try {
|
||||||
|
// Get the weekly registry
|
||||||
|
const weeklyRegistry = getRegistry('weekly');
|
||||||
|
|
||||||
|
const metrics = await weeklyRegistry.metrics();
|
||||||
|
event.node.res.setHeader('Content-Type', weeklyRegistry.contentType);
|
||||||
|
return metrics;
|
||||||
|
} catch (error) {
|
||||||
|
log.error('Error in weekly metrics endpoint:', {
|
||||||
|
evt: 'metrics_error',
|
||||||
|
error: error instanceof Error ? error.message : String(error),
|
||||||
|
});
|
||||||
|
throw createError({
|
||||||
|
statusCode: 500,
|
||||||
|
message: error instanceof Error ? error.message : 'Failed to collect weekly metrics',
|
||||||
|
});
|
||||||
|
}
|
||||||
|
});
|
||||||
48
server/tasks/jobs/clear-metrics/daily.ts
Normal file
48
server/tasks/jobs/clear-metrics/daily.ts
Normal file
|
|
@ -0,0 +1,48 @@
|
||||||
|
import { defineTask } from '#imports';
|
||||||
|
import { scopedLogger } from '../../../utils/logger';
|
||||||
|
import { setupMetrics } from '../../../utils/metrics';
|
||||||
|
|
||||||
|
const logger = scopedLogger('tasks:clear-metrics:daily');
|
||||||
|
|
||||||
|
export default defineTask({
|
||||||
|
meta: {
|
||||||
|
name: "jobs:clear-metrics:daily",
|
||||||
|
description: "Clear daily metrics at midnight",
|
||||||
|
},
|
||||||
|
async run() {
|
||||||
|
logger.info("Clearing daily metrics");
|
||||||
|
const startTime = Date.now();
|
||||||
|
|
||||||
|
try {
|
||||||
|
// Clear and reinitialize daily metrics
|
||||||
|
if (global.metrics_daily) {
|
||||||
|
global.metrics_daily.clear();
|
||||||
|
await setupMetrics('daily');
|
||||||
|
} else {
|
||||||
|
await setupMetrics('daily');
|
||||||
|
}
|
||||||
|
|
||||||
|
const executionTime = Date.now() - startTime;
|
||||||
|
logger.info(`Daily metrics cleared in ${executionTime}ms`);
|
||||||
|
|
||||||
|
return {
|
||||||
|
result: {
|
||||||
|
status: "success",
|
||||||
|
message: "Successfully cleared daily metrics",
|
||||||
|
executionTimeMs: executionTime,
|
||||||
|
timestamp: new Date().toISOString()
|
||||||
|
}
|
||||||
|
};
|
||||||
|
} catch (error) {
|
||||||
|
logger.error("Error clearing daily metrics:", { error: error.message });
|
||||||
|
return {
|
||||||
|
result: {
|
||||||
|
status: "error",
|
||||||
|
message: error.message || "An error occurred clearing daily metrics",
|
||||||
|
executionTimeMs: Date.now() - startTime,
|
||||||
|
timestamp: new Date().toISOString()
|
||||||
|
}
|
||||||
|
};
|
||||||
|
}
|
||||||
|
},
|
||||||
|
});
|
||||||
48
server/tasks/jobs/clear-metrics/monthly.ts
Normal file
48
server/tasks/jobs/clear-metrics/monthly.ts
Normal file
|
|
@ -0,0 +1,48 @@
|
||||||
|
import { defineTask } from '#imports';
|
||||||
|
import { scopedLogger } from '../../../utils/logger';
|
||||||
|
import { setupMetrics } from '../../../utils/metrics';
|
||||||
|
|
||||||
|
const logger = scopedLogger('tasks:clear-metrics:monthly');
|
||||||
|
|
||||||
|
export default defineTask({
|
||||||
|
meta: {
|
||||||
|
name: "jobs:clear-metrics:monthly",
|
||||||
|
description: "Clear monthly metrics on the 1st of each month at midnight",
|
||||||
|
},
|
||||||
|
async run() {
|
||||||
|
logger.info("Clearing monthly metrics");
|
||||||
|
const startTime = Date.now();
|
||||||
|
|
||||||
|
try {
|
||||||
|
// Clear and reinitialize monthly metrics
|
||||||
|
if (global.metrics_monthly) {
|
||||||
|
global.metrics_monthly.clear();
|
||||||
|
await setupMetrics('monthly');
|
||||||
|
} else {
|
||||||
|
await setupMetrics('monthly');
|
||||||
|
}
|
||||||
|
|
||||||
|
const executionTime = Date.now() - startTime;
|
||||||
|
logger.info(`Monthly metrics cleared in ${executionTime}ms`);
|
||||||
|
|
||||||
|
return {
|
||||||
|
result: {
|
||||||
|
status: "success",
|
||||||
|
message: "Successfully cleared monthly metrics",
|
||||||
|
executionTimeMs: executionTime,
|
||||||
|
timestamp: new Date().toISOString()
|
||||||
|
}
|
||||||
|
};
|
||||||
|
} catch (error) {
|
||||||
|
logger.error("Error clearing monthly metrics:", { error: error.message });
|
||||||
|
return {
|
||||||
|
result: {
|
||||||
|
status: "error",
|
||||||
|
message: error.message || "An error occurred clearing monthly metrics",
|
||||||
|
executionTimeMs: Date.now() - startTime,
|
||||||
|
timestamp: new Date().toISOString()
|
||||||
|
}
|
||||||
|
};
|
||||||
|
}
|
||||||
|
},
|
||||||
|
});
|
||||||
48
server/tasks/jobs/clear-metrics/weekly.ts
Normal file
48
server/tasks/jobs/clear-metrics/weekly.ts
Normal file
|
|
@ -0,0 +1,48 @@
|
||||||
|
import { defineTask } from '#imports';
|
||||||
|
import { scopedLogger } from '../../../utils/logger';
|
||||||
|
import { setupMetrics } from '../../../utils/metrics';
|
||||||
|
|
||||||
|
const logger = scopedLogger('tasks:clear-metrics:weekly');
|
||||||
|
|
||||||
|
export default defineTask({
|
||||||
|
meta: {
|
||||||
|
name: "jobs:clear-metrics:weekly",
|
||||||
|
description: "Clear weekly metrics every Sunday at midnight",
|
||||||
|
},
|
||||||
|
async run() {
|
||||||
|
logger.info("Clearing weekly metrics");
|
||||||
|
const startTime = Date.now();
|
||||||
|
|
||||||
|
try {
|
||||||
|
// Clear and reinitialize weekly metrics
|
||||||
|
if (global.metrics_weekly) {
|
||||||
|
global.metrics_weekly.clear();
|
||||||
|
await setupMetrics('weekly');
|
||||||
|
} else {
|
||||||
|
await setupMetrics('weekly');
|
||||||
|
}
|
||||||
|
|
||||||
|
const executionTime = Date.now() - startTime;
|
||||||
|
logger.info(`Weekly metrics cleared in ${executionTime}ms`);
|
||||||
|
|
||||||
|
return {
|
||||||
|
result: {
|
||||||
|
status: "success",
|
||||||
|
message: "Successfully cleared weekly metrics",
|
||||||
|
executionTimeMs: executionTime,
|
||||||
|
timestamp: new Date().toISOString()
|
||||||
|
}
|
||||||
|
};
|
||||||
|
} catch (error) {
|
||||||
|
logger.error("Error clearing weekly metrics:", { error: error.message });
|
||||||
|
return {
|
||||||
|
result: {
|
||||||
|
status: "error",
|
||||||
|
message: error.message || "An error occurred clearing weekly metrics",
|
||||||
|
executionTimeMs: Date.now() - startTime,
|
||||||
|
timestamp: new Date().toISOString()
|
||||||
|
}
|
||||||
|
};
|
||||||
|
}
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
@ -1,4 +1,4 @@
|
||||||
import { Counter, register, collectDefaultMetrics, Histogram, Summary } from 'prom-client';
|
import { Counter, register, collectDefaultMetrics, Histogram, Summary, Registry } from 'prom-client';
|
||||||
import { prisma } from './prisma';
|
import { prisma } from './prisma';
|
||||||
import { scopedLogger } from '~/utils/logger';
|
import { scopedLogger } from '~/utils/logger';
|
||||||
import fs from 'fs';
|
import fs from 'fs';
|
||||||
|
|
@ -6,6 +6,24 @@ import path from 'path';
|
||||||
|
|
||||||
const log = scopedLogger('metrics');
|
const log = scopedLogger('metrics');
|
||||||
const METRICS_FILE = '.metrics.json';
|
const METRICS_FILE = '.metrics.json';
|
||||||
|
const METRICS_DAILY_FILE = '.metrics_daily.json';
|
||||||
|
const METRICS_WEEKLY_FILE = '.metrics_weekly.json';
|
||||||
|
const METRICS_MONTHLY_FILE = '.metrics_monthly.json';
|
||||||
|
|
||||||
|
// Global registries
|
||||||
|
const registries = {
|
||||||
|
default: register, // All-time metrics (never cleared)
|
||||||
|
daily: new Registry(),
|
||||||
|
weekly: new Registry(),
|
||||||
|
monthly: new Registry()
|
||||||
|
};
|
||||||
|
|
||||||
|
// Expose the registries on global for tasks to access
|
||||||
|
if (typeof global !== 'undefined') {
|
||||||
|
global.metrics_daily = registries.daily;
|
||||||
|
global.metrics_weekly = registries.weekly;
|
||||||
|
global.metrics_monthly = registries.monthly;
|
||||||
|
}
|
||||||
|
|
||||||
export type Metrics = {
|
export type Metrics = {
|
||||||
user: Counter<'namespace'>;
|
user: Counter<'namespace'>;
|
||||||
|
|
@ -18,185 +36,230 @@ export type Metrics = {
|
||||||
httpRequestSummary: Summary<'method' | 'route' | 'status_code'>;
|
httpRequestSummary: Summary<'method' | 'route' | 'status_code'>;
|
||||||
};
|
};
|
||||||
|
|
||||||
let metrics: null | Metrics = null;
|
// Store metrics for each time period
|
||||||
|
const metricsStore: Record<string, Metrics | null> = {
|
||||||
|
default: null,
|
||||||
|
daily: null,
|
||||||
|
weekly: null,
|
||||||
|
monthly: null
|
||||||
|
};
|
||||||
|
|
||||||
export function getMetrics() {
|
export function getMetrics(interval: 'default' | 'daily' | 'weekly' | 'monthly' = 'default') {
|
||||||
if (!metrics) throw new Error('metrics not initialized');
|
if (!metricsStore[interval]) throw new Error(`metrics for ${interval} not initialized`);
|
||||||
return metrics;
|
return metricsStore[interval];
|
||||||
}
|
}
|
||||||
|
|
||||||
async function createMetrics(): Promise<Metrics> {
|
export function getRegistry(interval: 'default' | 'daily' | 'weekly' | 'monthly' = 'default') {
|
||||||
|
return registries[interval];
|
||||||
|
}
|
||||||
|
|
||||||
|
async function createMetrics(registry: Registry, interval: string): Promise<Metrics> {
|
||||||
const newMetrics = {
|
const newMetrics = {
|
||||||
user: new Counter({
|
user: new Counter({
|
||||||
name: 'mw_user_count',
|
name: `mw_user_count_${interval !== 'default' ? interval : ''}`,
|
||||||
help: 'Number of users by namespace',
|
help: `Number of users by namespace (${interval})`,
|
||||||
labelNames: ['namespace'],
|
labelNames: ['namespace'],
|
||||||
|
registers: [registry]
|
||||||
}),
|
}),
|
||||||
captchaSolves: new Counter({
|
captchaSolves: new Counter({
|
||||||
name: 'mw_captcha_solves',
|
name: `mw_captcha_solves_${interval !== 'default' ? interval : ''}`,
|
||||||
help: 'Number of captcha solves by success status',
|
help: `Number of captcha solves by success status (${interval})`,
|
||||||
labelNames: ['success'],
|
labelNames: ['success'],
|
||||||
|
registers: [registry]
|
||||||
}),
|
}),
|
||||||
providerHostnames: new Counter({
|
providerHostnames: new Counter({
|
||||||
name: 'mw_provider_hostname_count',
|
name: `mw_provider_hostname_count_${interval !== 'default' ? interval : ''}`,
|
||||||
help: 'Number of requests by provider hostname',
|
help: `Number of requests by provider hostname (${interval})`,
|
||||||
labelNames: ['hostname'],
|
labelNames: ['hostname'],
|
||||||
|
registers: [registry]
|
||||||
}),
|
}),
|
||||||
providerStatuses: new Counter({
|
providerStatuses: new Counter({
|
||||||
name: 'mw_provider_status_count',
|
name: `mw_provider_status_count_${interval !== 'default' ? interval : ''}`,
|
||||||
help: 'Number of provider requests by status',
|
help: `Number of provider requests by status (${interval})`,
|
||||||
labelNames: ['provider_id', 'status'],
|
labelNames: ['provider_id', 'status'],
|
||||||
|
registers: [registry]
|
||||||
}),
|
}),
|
||||||
watchMetrics: new Counter({
|
watchMetrics: new Counter({
|
||||||
name: 'mw_media_watch_count',
|
name: `mw_media_watch_count_${interval !== 'default' ? interval : ''}`,
|
||||||
help: 'Number of media watch events',
|
help: `Number of media watch events (${interval})`,
|
||||||
labelNames: ['title', 'tmdb_full_id', 'provider_id', 'success'],
|
labelNames: ['title', 'tmdb_full_id', 'provider_id', 'success'],
|
||||||
|
registers: [registry]
|
||||||
}),
|
}),
|
||||||
toolMetrics: new Counter({
|
toolMetrics: new Counter({
|
||||||
name: 'mw_provider_tool_count',
|
name: `mw_provider_tool_count_${interval !== 'default' ? interval : ''}`,
|
||||||
help: 'Number of provider tool usages',
|
help: `Number of provider tool usages (${interval})`,
|
||||||
labelNames: ['tool'],
|
labelNames: ['tool'],
|
||||||
|
registers: [registry]
|
||||||
}),
|
}),
|
||||||
httpRequestDuration: new Histogram({
|
httpRequestDuration: new Histogram({
|
||||||
name: 'http_request_duration_seconds',
|
name: `http_request_duration_seconds_${interval !== 'default' ? interval : ''}`,
|
||||||
help: 'request duration in seconds',
|
help: `request duration in seconds (${interval})`,
|
||||||
labelNames: ['method', 'route', 'status_code'],
|
labelNames: ['method', 'route', 'status_code'],
|
||||||
buckets: [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10],
|
buckets: [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10],
|
||||||
|
registers: [registry]
|
||||||
}),
|
}),
|
||||||
httpRequestSummary: new Summary({
|
httpRequestSummary: new Summary({
|
||||||
name: 'http_request_summary_seconds',
|
name: `http_request_summary_seconds_${interval !== 'default' ? interval : ''}`,
|
||||||
help: 'request duration in seconds summary',
|
help: `request duration in seconds summary (${interval})`,
|
||||||
labelNames: ['method', 'route', 'status_code'],
|
labelNames: ['method', 'route', 'status_code'],
|
||||||
percentiles: [0.01, 0.05, 0.5, 0.9, 0.95, 0.99, 0.999],
|
percentiles: [0.01, 0.05, 0.5, 0.9, 0.95, 0.99, 0.999],
|
||||||
|
registers: [registry]
|
||||||
}),
|
}),
|
||||||
};
|
};
|
||||||
|
|
||||||
// Register all metrics with the Prometheus registry
|
|
||||||
register.registerMetric(newMetrics.user);
|
|
||||||
register.registerMetric(newMetrics.captchaSolves);
|
|
||||||
register.registerMetric(newMetrics.providerHostnames);
|
|
||||||
register.registerMetric(newMetrics.providerStatuses);
|
|
||||||
register.registerMetric(newMetrics.watchMetrics);
|
|
||||||
register.registerMetric(newMetrics.toolMetrics);
|
|
||||||
register.registerMetric(newMetrics.httpRequestDuration);
|
|
||||||
register.registerMetric(newMetrics.httpRequestSummary);
|
|
||||||
|
|
||||||
return newMetrics;
|
return newMetrics;
|
||||||
}
|
}
|
||||||
|
|
||||||
async function saveMetricsToFile() {
|
async function saveMetricsToFile(interval: string = 'default') {
|
||||||
try {
|
try {
|
||||||
if (!metrics) return;
|
const registry = registries[interval];
|
||||||
|
if (!registry) return;
|
||||||
|
|
||||||
const metricsData = await register.getMetricsAsJSON();
|
const fileName = interval === 'default'
|
||||||
|
? METRICS_FILE
|
||||||
|
: interval === 'daily'
|
||||||
|
? METRICS_DAILY_FILE
|
||||||
|
: interval === 'weekly'
|
||||||
|
? METRICS_WEEKLY_FILE
|
||||||
|
: METRICS_MONTHLY_FILE;
|
||||||
|
|
||||||
|
const metricsData = await registry.getMetricsAsJSON();
|
||||||
const relevantMetrics = metricsData.filter(
|
const relevantMetrics = metricsData.filter(
|
||||||
metric => metric.name.startsWith('mw_') || metric.name === 'http_request_duration_seconds'
|
metric => metric.name.startsWith('mw_') || metric.name.startsWith('http_request')
|
||||||
);
|
);
|
||||||
|
|
||||||
fs.writeFileSync(METRICS_FILE, JSON.stringify(relevantMetrics, null, 2));
|
fs.writeFileSync(fileName, JSON.stringify(relevantMetrics, null, 2));
|
||||||
|
|
||||||
log.info('Metrics saved to file', { evt: 'metrics_saved' });
|
log.info(`${interval} metrics saved to file`, { evt: 'metrics_saved', interval });
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
log.error('Failed to save metrics', {
|
log.error(`Failed to save ${interval} metrics`, {
|
||||||
evt: 'save_metrics_error',
|
evt: 'save_metrics_error',
|
||||||
|
interval,
|
||||||
error: error instanceof Error ? error.message : String(error),
|
error: error instanceof Error ? error.message : String(error),
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async function loadMetricsFromFile(): Promise<any[]> {
|
async function loadMetricsFromFile(interval: string = 'default'): Promise<any[]> {
|
||||||
try {
|
try {
|
||||||
if (!fs.existsSync(METRICS_FILE)) {
|
const fileName = interval === 'default'
|
||||||
log.info('No saved metrics found', { evt: 'no_saved_metrics' });
|
? METRICS_FILE
|
||||||
|
: interval === 'daily'
|
||||||
|
? METRICS_DAILY_FILE
|
||||||
|
: interval === 'weekly'
|
||||||
|
? METRICS_WEEKLY_FILE
|
||||||
|
: METRICS_MONTHLY_FILE;
|
||||||
|
|
||||||
|
if (!fs.existsSync(fileName)) {
|
||||||
|
log.info(`No saved ${interval} metrics found`, { evt: 'no_saved_metrics', interval });
|
||||||
return [];
|
return [];
|
||||||
}
|
}
|
||||||
|
|
||||||
const data = fs.readFileSync(METRICS_FILE, 'utf8');
|
const data = fs.readFileSync(fileName, 'utf8');
|
||||||
const savedMetrics = JSON.parse(data);
|
const savedMetrics = JSON.parse(data);
|
||||||
log.info('Loaded saved metrics', {
|
log.info(`Loaded saved ${interval} metrics`, {
|
||||||
evt: 'metrics_loaded',
|
evt: 'metrics_loaded',
|
||||||
|
interval,
|
||||||
count: savedMetrics.length,
|
count: savedMetrics.length,
|
||||||
});
|
});
|
||||||
return savedMetrics;
|
return savedMetrics;
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
log.error('Failed to load metrics', {
|
log.error(`Failed to load ${interval} metrics`, {
|
||||||
evt: 'load_metrics_error',
|
evt: 'load_metrics_error',
|
||||||
|
interval,
|
||||||
error: error instanceof Error ? error.message : String(error),
|
error: error instanceof Error ? error.message : String(error),
|
||||||
});
|
});
|
||||||
return [];
|
return [];
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Periodically save metrics
|
// Periodically save all metrics
|
||||||
const SAVE_INTERVAL = 60000; // Save every minute
|
const SAVE_INTERVAL = 60000; // Save every minute
|
||||||
setInterval(saveMetricsToFile, SAVE_INTERVAL);
|
setInterval(() => {
|
||||||
|
Object.keys(registries).forEach(interval => {
|
||||||
|
saveMetricsToFile(interval);
|
||||||
|
});
|
||||||
|
}, SAVE_INTERVAL);
|
||||||
|
|
||||||
// Save metrics on process exit
|
// Save metrics on process exit
|
||||||
process.on('SIGTERM', async () => {
|
process.on('SIGTERM', async () => {
|
||||||
log.info('Saving metrics before exit...', { evt: 'exit_save' });
|
log.info('Saving all metrics before exit...', { evt: 'exit_save' });
|
||||||
await saveMetricsToFile();
|
for (const interval of Object.keys(registries)) {
|
||||||
|
await saveMetricsToFile(interval);
|
||||||
|
}
|
||||||
process.exit(0);
|
process.exit(0);
|
||||||
});
|
});
|
||||||
|
|
||||||
process.on('SIGINT', async () => {
|
process.on('SIGINT', async () => {
|
||||||
log.info('Saving metrics before exit...', { evt: 'exit_save' });
|
log.info('Saving all metrics before exit...', { evt: 'exit_save' });
|
||||||
await saveMetricsToFile();
|
for (const interval of Object.keys(registries)) {
|
||||||
|
await saveMetricsToFile(interval);
|
||||||
|
}
|
||||||
process.exit(0);
|
process.exit(0);
|
||||||
});
|
});
|
||||||
|
|
||||||
export async function setupMetrics() {
|
export async function setupMetrics(interval: 'default' | 'daily' | 'weekly' | 'monthly' = 'default') {
|
||||||
try {
|
try {
|
||||||
log.info('Setting up metrics...', { evt: 'start' });
|
log.info(`Setting up ${interval} metrics...`, { evt: 'start', interval });
|
||||||
|
|
||||||
// Clear all existing metrics
|
const registry = registries[interval];
|
||||||
log.info('Clearing metrics registry...', { evt: 'clear' });
|
|
||||||
register.clear();
|
|
||||||
|
|
||||||
// Enable default Node.js metrics collection with appropriate settings
|
// Clear registry if it already exists
|
||||||
collectDefaultMetrics({
|
registry.clear();
|
||||||
register,
|
|
||||||
prefix: '', // No prefix to match the example output
|
// Enable default Node.js metrics collection for default registry only
|
||||||
eventLoopMonitoringPrecision: 1, // Ensure eventloop metrics are collected
|
if (interval === 'default') {
|
||||||
gcDurationBuckets: [0.001, 0.01, 0.1, 1, 2, 5], // Match the example buckets
|
collectDefaultMetrics({
|
||||||
});
|
register: registry,
|
||||||
|
prefix: '', // No prefix to match the example output
|
||||||
|
eventLoopMonitoringPrecision: 1, // Ensure eventloop metrics are collected
|
||||||
|
gcDurationBuckets: [0.001, 0.01, 0.1, 1, 2, 5], // Match the example buckets
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
// Create new metrics instance
|
// Create new metrics instance
|
||||||
metrics = await createMetrics();
|
metricsStore[interval] = await createMetrics(registry, interval);
|
||||||
log.info('Created new metrics...', { evt: 'created' });
|
log.info(`Created new ${interval} metrics...`, { evt: 'created', interval });
|
||||||
|
|
||||||
// Load saved metrics
|
// Load saved metrics
|
||||||
const savedMetrics = await loadMetricsFromFile();
|
const savedMetrics = await loadMetricsFromFile(interval);
|
||||||
if (savedMetrics.length > 0) {
|
if (savedMetrics.length > 0) {
|
||||||
log.info('Restoring saved metrics...', { evt: 'restore_metrics' });
|
log.info(`Restoring saved ${interval} metrics...`, { evt: 'restore_metrics', interval });
|
||||||
savedMetrics.forEach(metric => {
|
savedMetrics.forEach(metric => {
|
||||||
if (metric.values) {
|
if (metric.values) {
|
||||||
metric.values.forEach(value => {
|
metric.values.forEach(value => {
|
||||||
switch (metric.name) {
|
const metrics = metricsStore[interval];
|
||||||
|
if (!metrics) return;
|
||||||
|
|
||||||
|
// Extract the base metric name without the interval suffix
|
||||||
|
const baseName = metric.name.replace(/_daily$|_weekly$|_monthly$/, '');
|
||||||
|
|
||||||
|
switch (baseName) {
|
||||||
case 'mw_user_count':
|
case 'mw_user_count':
|
||||||
metrics?.user.inc(value.labels, value.value);
|
metrics.user.inc(value.labels, value.value);
|
||||||
break;
|
break;
|
||||||
case 'mw_captcha_solves':
|
case 'mw_captcha_solves':
|
||||||
metrics?.captchaSolves.inc(value.labels, value.value);
|
metrics.captchaSolves.inc(value.labels, value.value);
|
||||||
break;
|
break;
|
||||||
case 'mw_provider_hostname_count':
|
case 'mw_provider_hostname_count':
|
||||||
metrics?.providerHostnames.inc(value.labels, value.value);
|
metrics.providerHostnames.inc(value.labels, value.value);
|
||||||
break;
|
break;
|
||||||
case 'mw_provider_status_count':
|
case 'mw_provider_status_count':
|
||||||
metrics?.providerStatuses.inc(value.labels, value.value);
|
metrics.providerStatuses.inc(value.labels, value.value);
|
||||||
break;
|
break;
|
||||||
case 'mw_media_watch_count':
|
case 'mw_media_watch_count':
|
||||||
metrics?.watchMetrics.inc(value.labels, value.value);
|
metrics.watchMetrics.inc(value.labels, value.value);
|
||||||
break;
|
break;
|
||||||
case 'mw_provider_tool_count':
|
case 'mw_provider_tool_count':
|
||||||
metrics?.toolMetrics.inc(value.labels, value.value);
|
metrics.toolMetrics.inc(value.labels, value.value);
|
||||||
break;
|
break;
|
||||||
case 'http_request_duration_seconds':
|
case 'http_request_duration_seconds':
|
||||||
// For histograms, special handling for sum and count
|
// For histograms, special handling for sum and count
|
||||||
if (
|
if (
|
||||||
value.metricName === 'http_request_duration_seconds_sum' ||
|
value.metricName === `http_request_duration_seconds_${interval !== 'default' ? interval + '_' : ''}sum` ||
|
||||||
value.metricName === 'http_request_duration_seconds_count'
|
value.metricName === `http_request_duration_seconds_${interval !== 'default' ? interval + '_' : ''}count`
|
||||||
) {
|
) {
|
||||||
metrics?.httpRequestDuration.observe(value.labels, value.value);
|
metrics.httpRequestDuration.observe(value.labels, value.value);
|
||||||
}
|
}
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
|
@ -206,112 +269,132 @@ export async function setupMetrics() {
|
||||||
}
|
}
|
||||||
|
|
||||||
// Initialize metrics with current data
|
// Initialize metrics with current data
|
||||||
log.info('Syncing up metrics...', { evt: 'sync' });
|
log.info(`Syncing up ${interval} metrics...`, { evt: 'sync', interval });
|
||||||
await updateMetrics();
|
await updateMetrics(interval);
|
||||||
log.info('Metrics initialized!', { evt: 'end' });
|
log.info(`${interval} metrics initialized!`, { evt: 'end', interval });
|
||||||
|
|
||||||
// Save initial state
|
// Save initial state
|
||||||
await saveMetricsToFile();
|
await saveMetricsToFile(interval);
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
log.error('Failed to setup metrics', {
|
log.error(`Failed to setup ${interval} metrics`, {
|
||||||
evt: 'setup_error',
|
evt: 'setup_error',
|
||||||
|
interval,
|
||||||
error: error instanceof Error ? error.message : String(error),
|
error: error instanceof Error ? error.message : String(error),
|
||||||
});
|
});
|
||||||
throw error;
|
throw error;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async function updateMetrics() {
|
async function updateMetrics(interval: 'default' | 'daily' | 'weekly' | 'monthly' = 'default') {
|
||||||
try {
|
try {
|
||||||
log.info('Fetching users from database...', { evt: 'update_metrics_start' });
|
log.info(`Fetching users from database for ${interval} metrics...`, { evt: 'update_metrics_start', interval });
|
||||||
|
|
||||||
const users = await prisma.users.groupBy({
|
const users = await prisma.users.groupBy({
|
||||||
by: ['namespace'],
|
by: ['namespace'],
|
||||||
_count: true,
|
_count: true,
|
||||||
});
|
});
|
||||||
|
|
||||||
log.info('Found users', { evt: 'users_found', count: users.length });
|
log.info('Found users', { evt: 'users_found', count: users.length, interval });
|
||||||
|
|
||||||
metrics?.user.reset();
|
const metrics = metricsStore[interval];
|
||||||
log.info('Reset user metrics counter', { evt: 'metrics_reset' });
|
if (!metrics) return;
|
||||||
|
|
||||||
|
metrics.user.reset();
|
||||||
|
log.info(`Reset user metrics counter for ${interval}`, { evt: 'metrics_reset', interval });
|
||||||
|
|
||||||
users.forEach(v => {
|
users.forEach(v => {
|
||||||
log.info('Incrementing user metric', {
|
log.info(`Incrementing user metric for ${interval}`, {
|
||||||
evt: 'increment_metric',
|
evt: 'increment_metric',
|
||||||
|
interval,
|
||||||
namespace: v.namespace,
|
namespace: v.namespace,
|
||||||
count: v._count,
|
count: v._count,
|
||||||
});
|
});
|
||||||
metrics?.user.inc({ namespace: v.namespace }, v._count);
|
metrics.user.inc({ namespace: v.namespace }, v._count);
|
||||||
});
|
});
|
||||||
|
|
||||||
log.info('Successfully updated metrics', { evt: 'update_metrics_complete' });
|
log.info(`Successfully updated ${interval} metrics`, { evt: 'update_metrics_complete', interval });
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
log.error('Failed to update metrics', {
|
log.error(`Failed to update ${interval} metrics`, {
|
||||||
evt: 'update_metrics_error',
|
evt: 'update_metrics_error',
|
||||||
|
interval,
|
||||||
error: error instanceof Error ? error.message : String(error),
|
error: error instanceof Error ? error.message : String(error),
|
||||||
});
|
});
|
||||||
throw error;
|
throw error;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Export function to record HTTP request duration
|
// Export function to record HTTP request duration for all registries
|
||||||
export function recordHttpRequest(
|
export function recordHttpRequest(
|
||||||
method: string,
|
method: string,
|
||||||
route: string,
|
route: string,
|
||||||
statusCode: number,
|
statusCode: number,
|
||||||
duration: number
|
duration: number
|
||||||
) {
|
) {
|
||||||
if (!metrics) return;
|
|
||||||
|
|
||||||
const labels = {
|
const labels = {
|
||||||
method,
|
method,
|
||||||
route,
|
route,
|
||||||
status_code: statusCode.toString(),
|
status_code: statusCode.toString(),
|
||||||
};
|
};
|
||||||
|
|
||||||
// Record in both histogram and summary
|
// Record in all active registries
|
||||||
metrics.httpRequestDuration.observe(labels, duration);
|
Object.entries(metricsStore).forEach(([interval, metrics]) => {
|
||||||
metrics.httpRequestSummary.observe(labels, duration);
|
if (!metrics) return;
|
||||||
|
|
||||||
|
// Record in both histogram and summary
|
||||||
|
metrics.httpRequestDuration.observe(labels, duration);
|
||||||
|
metrics.httpRequestSummary.observe(labels, duration);
|
||||||
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
// Functions to match previous backend API
|
// Functions to match previous backend API - record in all registries
|
||||||
export function recordProviderMetrics(items: any[], hostname: string, tool?: string) {
|
export function recordProviderMetrics(items: any[], hostname: string, tool?: string) {
|
||||||
if (!metrics) return;
|
Object.values(metricsStore).forEach(metrics => {
|
||||||
|
if (!metrics) return;
|
||||||
|
|
||||||
// Record hostname once per request
|
// Record hostname once per request
|
||||||
metrics.providerHostnames.inc({ hostname });
|
metrics.providerHostnames.inc({ hostname });
|
||||||
|
|
||||||
// Record status metrics for each item
|
// Record status metrics for each item
|
||||||
items.forEach(item => {
|
items.forEach(item => {
|
||||||
// Record provider status
|
// Record provider status
|
||||||
metrics.providerStatuses.inc({
|
metrics.providerStatuses.inc({
|
||||||
provider_id: item.embedId ?? item.providerId,
|
provider_id: item.embedId ?? item.providerId,
|
||||||
status: item.status,
|
status: item.status,
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
// Reverse items to get the last one, and find the last successful item
|
||||||
|
const itemList = [...items];
|
||||||
|
itemList.reverse();
|
||||||
|
const lastSuccessfulItem = items.find(v => v.status === 'success');
|
||||||
|
const lastItem = itemList[0];
|
||||||
|
|
||||||
|
// Record watch metrics only for the last item
|
||||||
|
if (lastItem) {
|
||||||
|
metrics.watchMetrics.inc({
|
||||||
|
tmdb_full_id: lastItem.type + '-' + lastItem.tmdbId,
|
||||||
|
provider_id: lastSuccessfulItem?.providerId ?? lastItem.providerId,
|
||||||
|
title: lastItem.title,
|
||||||
|
success: (!!lastSuccessfulItem).toString(),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
// Record tool metrics
|
||||||
|
if (tool) {
|
||||||
|
metrics.toolMetrics.inc({ tool });
|
||||||
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
// Reverse items to get the last one, and find the last successful item
|
|
||||||
const itemList = [...items];
|
|
||||||
itemList.reverse();
|
|
||||||
const lastSuccessfulItem = items.find(v => v.status === 'success');
|
|
||||||
const lastItem = itemList[0];
|
|
||||||
|
|
||||||
// Record watch metrics only for the last item
|
|
||||||
if (lastItem) {
|
|
||||||
metrics.watchMetrics.inc({
|
|
||||||
tmdb_full_id: lastItem.type + '-' + lastItem.tmdbId,
|
|
||||||
provider_id: lastSuccessfulItem?.providerId ?? lastItem.providerId,
|
|
||||||
title: lastItem.title,
|
|
||||||
success: (!!lastSuccessfulItem).toString(),
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
// Record tool metrics
|
|
||||||
if (tool) {
|
|
||||||
metrics.toolMetrics.inc({ tool });
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
export function recordCaptchaMetrics(success: boolean) {
|
export function recordCaptchaMetrics(success: boolean) {
|
||||||
metrics?.captchaSolves.inc({ success: success.toString() });
|
Object.values(metricsStore).forEach(metrics => {
|
||||||
|
metrics?.captchaSolves.inc({ success: success.toString() });
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
// Initialize all metrics registries on startup
|
||||||
|
export async function initializeAllMetrics() {
|
||||||
|
for (const interval of Object.keys(registries) as Array<'default' | 'daily' | 'weekly' | 'monthly'>) {
|
||||||
|
await setupMetrics(interval);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue