diff --git a/src/hooks/useMetadata.ts b/src/hooks/useMetadata.ts index be819c39..ca7b4d3e 100644 --- a/src/hooks/useMetadata.ts +++ b/src/hooks/useMetadata.ts @@ -107,6 +107,28 @@ interface UseMetadataReturn { imdbId: string | null; scraperStatuses: ScraperStatus[]; activeFetchingScrapers: string[]; + clearScraperCache: () => Promise; + invalidateScraperCache: (scraperId: string) => Promise; + invalidateContentCache: (type: string, tmdbId: string, season?: number, episode?: number) => Promise; + getScraperCacheStats: () => Promise<{ + local: { + totalEntries: number; + totalSize: number; + oldestEntry: number | null; + newestEntry: number | null; + }; + global: { + totalEntries: number; + totalSize: number; + oldestEntry: number | null; + newestEntry: number | null; + hitRate: number; + }; + combined: { + totalEntries: number; + hitRate: number; + }; + }>; } export const useMetadata = ({ id, type, addonId }: UseMetadataProps): UseMetadataReturn => { @@ -1484,6 +1506,23 @@ export const useMetadata = ({ id, type, addonId }: UseMetadataProps): UseMetadat }; }, [cleanupStreams]); + // Cache management methods + const clearScraperCache = useCallback(async () => { + await localScraperService.clearScraperCache(); + }, []); + + const invalidateScraperCache = useCallback(async (scraperId: string) => { + await localScraperService.invalidateScraperCache(scraperId); + }, []); + + const invalidateContentCache = useCallback(async (type: string, tmdbId: string, season?: number, episode?: number) => { + await localScraperService.invalidateContentCache(type, tmdbId, season, episode); + }, []); + + const getScraperCacheStats = useCallback(async () => { + return await localScraperService.getCacheStats(); + }, []); + return { metadata, loading, @@ -1517,5 +1556,9 @@ export const useMetadata = ({ id, type, addonId }: UseMetadataProps): UseMetadat imdbId, scraperStatuses, activeFetchingScrapers, + clearScraperCache, + invalidateScraperCache, + invalidateContentCache, + getScraperCacheStats, }; }; \ No newline at end of file diff --git a/src/services/hybridCacheService.ts b/src/services/hybridCacheService.ts new file mode 100644 index 00000000..e93d418e --- /dev/null +++ b/src/services/hybridCacheService.ts @@ -0,0 +1,379 @@ +import { localScraperCacheService, CachedScraperResult } from './localScraperCacheService'; +import { supabaseGlobalCacheService, GlobalCachedScraperResult } from './supabaseGlobalCacheService'; +import { logger } from '../utils/logger'; +import { Stream } from '../types/streams'; + +export interface HybridCacheResult { + validResults: Array; + expiredScrapers: string[]; + allExpired: boolean; + source: 'local' | 'global' | 'hybrid'; +} + +export interface HybridCacheStats { + local: { + totalEntries: number; + totalSize: number; + oldestEntry: number | null; + newestEntry: number | null; + }; + global: { + totalEntries: number; + totalSize: number; + oldestEntry: number | null; + newestEntry: number | null; + hitRate: number; + }; + combined: { + totalEntries: number; + hitRate: number; + }; +} + +class HybridCacheService { + private static instance: HybridCacheService; + private readonly ENABLE_GLOBAL_CACHE = true; // Can be made configurable + private readonly FALLBACK_TO_LOCAL = true; // Fallback to local if global fails + + private constructor() {} + + public static getInstance(): HybridCacheService { + if (!HybridCacheService.instance) { + HybridCacheService.instance = new HybridCacheService(); + } + return HybridCacheService.instance; + } + + /** + * Get cached results with hybrid approach (global first, then local) + */ + async getCachedResults( + type: string, + tmdbId: string, + season?: number, + episode?: number + ): Promise { + try { + // Try global cache first if enabled + if (this.ENABLE_GLOBAL_CACHE) { + try { + const globalResults = await supabaseGlobalCacheService.getCachedResults(type, tmdbId, season, episode); + + if (globalResults.validResults.length > 0) { + logger.log(`[HybridCache] Using global cache: ${globalResults.validResults.length} results`); + return { + ...globalResults, + source: 'global' + }; + } + } catch (error) { + logger.warn('[HybridCache] Global cache failed, falling back to local:', error); + } + } + + // Fallback to local cache + if (this.FALLBACK_TO_LOCAL) { + const localResults = await localScraperCacheService.getCachedResults(type, tmdbId, season, episode); + + if (localResults.validResults.length > 0) { + logger.log(`[HybridCache] Using local cache: ${localResults.validResults.length} results`); + return { + ...localResults, + source: 'local' + }; + } + } + + // No valid results found + return { + validResults: [], + expiredScrapers: [], + allExpired: true, + source: 'hybrid' + }; + + } catch (error) { + logger.error('[HybridCache] Error getting cached results:', error); + return { + validResults: [], + expiredScrapers: [], + allExpired: true, + source: 'hybrid' + }; + } + } + + /** + * Cache results in both local and global cache + */ + async cacheResults( + type: string, + tmdbId: string, + results: Array<{ + scraperId: string; + scraperName: string; + streams: Stream[] | null; + error: Error | null; + }>, + season?: number, + episode?: number + ): Promise { + try { + // Cache in local storage first (fastest) + const localPromises = results.map(result => + localScraperCacheService.cacheScraperResult( + type, tmdbId, result.scraperId, result.scraperName, + result.streams, result.error, season, episode + ) + ); + await Promise.all(localPromises); + + // Cache in global storage (shared across users) + if (this.ENABLE_GLOBAL_CACHE) { + try { + await supabaseGlobalCacheService.cacheResults(type, tmdbId, results, season, episode); + logger.log(`[HybridCache] Cached ${results.length} results in both local and global cache`); + } catch (error) { + logger.warn('[HybridCache] Failed to cache in global storage:', error); + // Local cache succeeded, so we continue + } + } + + } catch (error) { + logger.error('[HybridCache] Error caching results:', error); + } + } + + /** + * Cache a single scraper result + */ + async cacheScraperResult( + type: string, + tmdbId: string, + scraperId: string, + scraperName: string, + streams: Stream[] | null, + error: Error | null, + season?: number, + episode?: number + ): Promise { + await this.cacheResults(type, tmdbId, [{ + scraperId, + scraperName, + streams, + error + }], season, episode); + } + + /** + * Get list of scrapers that need to be re-run + */ + async getScrapersToRerun( + type: string, + tmdbId: string, + availableScrapers: Array<{ id: string; name: string }>, + season?: number, + episode?: number + ): Promise { + const { validResults, expiredScrapers } = await this.getCachedResults(type, tmdbId, season, episode); + + const validScraperIds = new Set(validResults.map(r => r.scraperId)); + const expiredScraperIds = new Set(expiredScrapers); + + // Return scrapers that are either expired or not cached + const scrapersToRerun = availableScrapers + .filter(scraper => + !validScraperIds.has(scraper.id) || expiredScraperIds.has(scraper.id) + ) + .map(scraper => scraper.id); + + logger.log(`[HybridCache] Scrapers to re-run: ${scrapersToRerun.join(', ')}`); + + return scrapersToRerun; + } + + /** + * Get all valid cached streams + */ + async getCachedStreams( + type: string, + tmdbId: string, + season?: number, + episode?: number + ): Promise { + const { validResults } = await this.getCachedResults(type, tmdbId, season, episode); + + // Flatten all valid streams + const allStreams: Stream[] = []; + for (const result of validResults) { + if (result.success && result.streams) { + allStreams.push(...result.streams); + } + } + + return allStreams; + } + + /** + * Invalidate cache for specific content + */ + async invalidateContent( + type: string, + tmdbId: string, + season?: number, + episode?: number + ): Promise { + try { + // Invalidate both local and global cache + const promises = [ + localScraperCacheService.invalidateContent(type, tmdbId, season, episode) + ]; + + if (this.ENABLE_GLOBAL_CACHE) { + promises.push( + supabaseGlobalCacheService.invalidateContent(type, tmdbId, season, episode) + ); + } + + await Promise.all(promises); + logger.log(`[HybridCache] Invalidated cache for ${type}:${tmdbId}`); + } catch (error) { + logger.error('[HybridCache] Error invalidating cache:', error); + } + } + + /** + * Invalidate cache for specific scraper + */ + async invalidateScraper(scraperId: string): Promise { + try { + // Invalidate both local and global cache + const promises = [ + localScraperCacheService.invalidateScraper(scraperId) + ]; + + if (this.ENABLE_GLOBAL_CACHE) { + promises.push( + supabaseGlobalCacheService.invalidateScraper(scraperId) + ); + } + + await Promise.all(promises); + logger.log(`[HybridCache] Invalidated cache for scraper ${scraperId}`); + } catch (error) { + logger.error('[HybridCache] Error invalidating scraper cache:', error); + } + } + + /** + * Clear all cached results + */ + async clearAllCache(): Promise { + try { + // Clear both local and global cache + const promises = [ + localScraperCacheService.clearAllCache() + ]; + + if (this.ENABLE_GLOBAL_CACHE) { + promises.push( + supabaseGlobalCacheService.clearAllCache() + ); + } + + await Promise.all(promises); + logger.log('[HybridCache] Cleared all cache (local and global)'); + } catch (error) { + logger.error('[HybridCache] Error clearing cache:', error); + } + } + + /** + * Get combined cache statistics + */ + async getCacheStats(): Promise { + try { + const [localStats, globalStats] = await Promise.all([ + localScraperCacheService.getCacheStats(), + this.ENABLE_GLOBAL_CACHE ? supabaseGlobalCacheService.getCacheStats() : Promise.resolve({ + totalEntries: 0, + totalSize: 0, + oldestEntry: null, + newestEntry: null, + hitRate: 0 + }) + ]); + + return { + local: localStats, + global: globalStats, + combined: { + totalEntries: localStats.totalEntries + globalStats.totalEntries, + hitRate: globalStats.hitRate // Global cache hit rate is more meaningful + } + }; + } catch (error) { + logger.error('[HybridCache] Error getting cache stats:', error); + return { + local: { totalEntries: 0, totalSize: 0, oldestEntry: null, newestEntry: null }, + global: { totalEntries: 0, totalSize: 0, oldestEntry: null, newestEntry: null, hitRate: 0 }, + combined: { totalEntries: 0, hitRate: 0 } + }; + } + } + + /** + * Clean up old entries in both caches + */ + async cleanupOldEntries(): Promise { + try { + const promises = [ + localScraperCacheService.clearAllCache() // Local cache handles cleanup automatically + ]; + + if (this.ENABLE_GLOBAL_CACHE) { + promises.push( + supabaseGlobalCacheService.cleanupOldEntries() + ); + } + + await Promise.all(promises); + logger.log('[HybridCache] Cleaned up old entries'); + } catch (error) { + logger.error('[HybridCache] Error cleaning up old entries:', error); + } + } + + /** + * Get cache configuration + */ + getConfig(): { + enableGlobalCache: boolean; + fallbackToLocal: boolean; + } { + return { + enableGlobalCache: this.ENABLE_GLOBAL_CACHE, + fallbackToLocal: this.FALLBACK_TO_LOCAL + }; + } + + /** + * Update cache configuration + */ + updateConfig(config: { + enableGlobalCache?: boolean; + fallbackToLocal?: boolean; + }): void { + if (config.enableGlobalCache !== undefined) { + (this as any).ENABLE_GLOBAL_CACHE = config.enableGlobalCache; + } + if (config.fallbackToLocal !== undefined) { + (this as any).FALLBACK_TO_LOCAL = config.fallbackToLocal; + } + + logger.log('[HybridCache] Configuration updated:', this.getConfig()); + } +} + +export const hybridCacheService = HybridCacheService.getInstance(); +export default hybridCacheService; diff --git a/src/services/localScraperCacheService.ts b/src/services/localScraperCacheService.ts new file mode 100644 index 00000000..67ccf1fd --- /dev/null +++ b/src/services/localScraperCacheService.ts @@ -0,0 +1,425 @@ +import AsyncStorage from '@react-native-async-storage/async-storage'; +import { logger } from '../utils/logger'; +import { Stream } from '../types/streams'; + +export interface CachedScraperResult { + streams: Stream[]; + timestamp: number; + success: boolean; + error?: string; + scraperId: string; + scraperName: string; +} + +export interface CachedContentResult { + contentKey: string; // e.g., "movie:123" or "tv:123:1:2" + results: CachedScraperResult[]; + timestamp: number; + ttl: number; +} + +class LocalScraperCacheService { + private static instance: LocalScraperCacheService; + private readonly CACHE_KEY_PREFIX = 'local-scraper-cache'; + private readonly DEFAULT_TTL_MS = 30 * 60 * 1000; // 30 minutes default TTL + private readonly MAX_CACHE_SIZE = 200; // Maximum number of cached content items + private readonly FAILED_RETRY_TTL_MS = 5 * 60 * 1000; // 5 minutes for failed scrapers + private readonly SUCCESS_TTL_MS = 60 * 60 * 1000; // 1 hour for successful scrapers + + private constructor() {} + + public static getInstance(): LocalScraperCacheService { + if (!LocalScraperCacheService.instance) { + LocalScraperCacheService.instance = new LocalScraperCacheService(); + } + return LocalScraperCacheService.instance; + } + + /** + * Generate cache key for content + */ + private getContentKey(type: string, tmdbId: string, season?: number, episode?: number): string { + if (season !== undefined && episode !== undefined) { + return `${type}:${tmdbId}:${season}:${episode}`; + } + return `${type}:${tmdbId}`; + } + + /** + * Generate AsyncStorage key for cached content + */ + private getStorageKey(contentKey: string): string { + return `${this.CACHE_KEY_PREFIX}:${contentKey}`; + } + + /** + * Check if cached result is still valid based on TTL + */ + private isCacheValid(timestamp: number, ttl: number): boolean { + return Date.now() - timestamp < ttl; + } + + /** + * Get cached results for content, filtering out expired results + */ + async getCachedResults( + type: string, + tmdbId: string, + season?: number, + episode?: number + ): Promise<{ + validResults: CachedScraperResult[]; + expiredScrapers: string[]; + allExpired: boolean; + }> { + try { + const contentKey = this.getContentKey(type, tmdbId, season, episode); + const storageKey = this.getStorageKey(contentKey); + + const cachedData = await AsyncStorage.getItem(storageKey); + if (!cachedData) { + return { + validResults: [], + expiredScrapers: [], + allExpired: true + }; + } + + const parsed: CachedContentResult = JSON.parse(cachedData); + + // Check if the entire cache entry is expired + if (!this.isCacheValid(parsed.timestamp, parsed.ttl)) { + // Remove expired entry + await AsyncStorage.removeItem(storageKey); + return { + validResults: [], + expiredScrapers: parsed.results.map(r => r.scraperId), + allExpired: true + }; + } + + // Filter valid results and identify expired scrapers + const validResults: CachedScraperResult[] = []; + const expiredScrapers: string[] = []; + + for (const result of parsed.results) { + // Use different TTL based on success/failure + const ttl = result.success ? this.SUCCESS_TTL_MS : this.FAILED_RETRY_TTL_MS; + + if (this.isCacheValid(result.timestamp, ttl)) { + validResults.push(result); + } else { + expiredScrapers.push(result.scraperId); + } + } + + logger.log(`[LocalScraperCache] Retrieved ${validResults.length} valid results, ${expiredScrapers.length} expired scrapers for ${contentKey}`); + + return { + validResults, + expiredScrapers, + allExpired: validResults.length === 0 + }; + + } catch (error) { + logger.error('[LocalScraperCache] Error getting cached results:', error); + return { + validResults: [], + expiredScrapers: [], + allExpired: true + }; + } + } + + /** + * Cache results for specific scrapers + */ + async cacheResults( + type: string, + tmdbId: string, + results: CachedScraperResult[], + season?: number, + episode?: number + ): Promise { + try { + const contentKey = this.getContentKey(type, tmdbId, season, episode); + const storageKey = this.getStorageKey(contentKey); + + // Get existing cached data + const existingData = await AsyncStorage.getItem(storageKey); + let cachedContent: CachedContentResult; + + if (existingData) { + cachedContent = JSON.parse(existingData); + + // Update existing results or add new ones + for (const newResult of results) { + const existingIndex = cachedContent.results.findIndex(r => r.scraperId === newResult.scraperId); + if (existingIndex >= 0) { + // Update existing result + cachedContent.results[existingIndex] = newResult; + } else { + // Add new result + cachedContent.results.push(newResult); + } + } + } else { + // Create new cache entry + cachedContent = { + contentKey, + results, + timestamp: Date.now(), + ttl: this.DEFAULT_TTL_MS + }; + } + + // Update timestamp + cachedContent.timestamp = Date.now(); + + // Store updated cache + await AsyncStorage.setItem(storageKey, JSON.stringify(cachedContent)); + + // Clean up old cache entries if we exceed the limit + await this.cleanupOldEntries(); + + logger.log(`[LocalScraperCache] Cached ${results.length} results for ${contentKey}`); + + } catch (error) { + logger.error('[LocalScraperCache] Error caching results:', error); + } + } + + /** + * Cache a single scraper result + */ + async cacheScraperResult( + type: string, + tmdbId: string, + scraperId: string, + scraperName: string, + streams: Stream[] | null, + error: Error | null, + season?: number, + episode?: number + ): Promise { + const result: CachedScraperResult = { + streams: streams || [], + timestamp: Date.now(), + success: !error && streams !== null, + error: error?.message, + scraperId, + scraperName + }; + + await this.cacheResults(type, tmdbId, [result], season, episode); + } + + /** + * Get list of scrapers that need to be re-run (expired or failed) + */ + async getScrapersToRerun( + type: string, + tmdbId: string, + availableScrapers: Array<{ id: string; name: string }>, + season?: number, + episode?: number + ): Promise { + const { validResults, expiredScrapers } = await this.getCachedResults(type, tmdbId, season, episode); + + const validScraperIds = new Set(validResults.map(r => r.scraperId)); + const expiredScraperIds = new Set(expiredScrapers); + + // Return scrapers that are either expired or not cached at all + const scrapersToRerun = availableScrapers + .filter(scraper => + !validScraperIds.has(scraper.id) || expiredScraperIds.has(scraper.id) + ) + .map(scraper => scraper.id); + + logger.log(`[LocalScraperCache] Scrapers to re-run: ${scrapersToRerun.join(', ')}`); + + return scrapersToRerun; + } + + /** + * Get all valid cached streams for content + */ + async getCachedStreams( + type: string, + tmdbId: string, + season?: number, + episode?: number + ): Promise { + const { validResults } = await this.getCachedResults(type, tmdbId, season, episode); + + // Flatten all valid streams + const allStreams: Stream[] = []; + for (const result of validResults) { + if (result.success && result.streams) { + allStreams.push(...result.streams); + } + } + + return allStreams; + } + + /** + * Invalidate cache for specific content + */ + async invalidateContent( + type: string, + tmdbId: string, + season?: number, + episode?: number + ): Promise { + try { + const contentKey = this.getContentKey(type, tmdbId, season, episode); + const storageKey = this.getStorageKey(contentKey); + + await AsyncStorage.removeItem(storageKey); + logger.log(`[LocalScraperCache] Invalidated cache for ${contentKey}`); + } catch (error) { + logger.error('[LocalScraperCache] Error invalidating cache:', error); + } + } + + /** + * Invalidate cache for specific scraper across all content + */ + async invalidateScraper(scraperId: string): Promise { + try { + const keys = await AsyncStorage.getAllKeys(); + const cacheKeys = keys.filter(key => key.startsWith(this.CACHE_KEY_PREFIX)); + + for (const key of cacheKeys) { + const cachedData = await AsyncStorage.getItem(key); + if (cachedData) { + const parsed: CachedContentResult = JSON.parse(cachedData); + + // Remove results from this scraper + parsed.results = parsed.results.filter(r => r.scraperId !== scraperId); + + if (parsed.results.length === 0) { + // Remove entire cache entry if no results left + await AsyncStorage.removeItem(key); + } else { + // Update cache with remaining results + await AsyncStorage.setItem(key, JSON.stringify(parsed)); + } + } + } + + logger.log(`[LocalScraperCache] Invalidated cache for scraper ${scraperId}`); + } catch (error) { + logger.error('[LocalScraperCache] Error invalidating scraper cache:', error); + } + } + + /** + * Clear all cached results + */ + async clearAllCache(): Promise { + try { + const keys = await AsyncStorage.getAllKeys(); + const cacheKeys = keys.filter(key => key.startsWith(this.CACHE_KEY_PREFIX)); + + await AsyncStorage.multiRemove(cacheKeys); + logger.log(`[LocalScraperCache] Cleared ${cacheKeys.length} cache entries`); + } catch (error) { + logger.error('[LocalScraperCache] Error clearing cache:', error); + } + } + + /** + * Clean up old cache entries to stay within size limit + */ + private async cleanupOldEntries(): Promise { + try { + const keys = await AsyncStorage.getAllKeys(); + const cacheKeys = keys.filter(key => key.startsWith(this.CACHE_KEY_PREFIX)); + + if (cacheKeys.length <= this.MAX_CACHE_SIZE) { + return; // No cleanup needed + } + + // Get all cache entries with their timestamps + const entriesWithTimestamps = await Promise.all( + cacheKeys.map(async (key) => { + const data = await AsyncStorage.getItem(key); + if (data) { + const parsed: CachedContentResult = JSON.parse(data); + return { key, timestamp: parsed.timestamp }; + } + return { key, timestamp: 0 }; + }) + ); + + // Sort by timestamp (oldest first) + entriesWithTimestamps.sort((a, b) => a.timestamp - b.timestamp); + + // Remove oldest entries + const entriesToRemove = entriesWithTimestamps.slice(0, cacheKeys.length - this.MAX_CACHE_SIZE); + const keysToRemove = entriesToRemove.map(entry => entry.key); + + if (keysToRemove.length > 0) { + await AsyncStorage.multiRemove(keysToRemove); + logger.log(`[LocalScraperCache] Cleaned up ${keysToRemove.length} old cache entries`); + } + + } catch (error) { + logger.error('[LocalScraperCache] Error cleaning up cache:', error); + } + } + + /** + * Get cache statistics + */ + async getCacheStats(): Promise<{ + totalEntries: number; + totalSize: number; + oldestEntry: number | null; + newestEntry: number | null; + }> { + try { + const keys = await AsyncStorage.getAllKeys(); + const cacheKeys = keys.filter(key => key.startsWith(this.CACHE_KEY_PREFIX)); + + let totalSize = 0; + let oldestTimestamp: number | null = null; + let newestTimestamp: number | null = null; + + for (const key of cacheKeys) { + const data = await AsyncStorage.getItem(key); + if (data) { + totalSize += data.length; + const parsed: CachedContentResult = JSON.parse(data); + + if (oldestTimestamp === null || parsed.timestamp < oldestTimestamp) { + oldestTimestamp = parsed.timestamp; + } + if (newestTimestamp === null || parsed.timestamp > newestTimestamp) { + newestTimestamp = parsed.timestamp; + } + } + } + + return { + totalEntries: cacheKeys.length, + totalSize, + oldestEntry: oldestTimestamp, + newestEntry: newestTimestamp + }; + } catch (error) { + logger.error('[LocalScraperCache] Error getting cache stats:', error); + return { + totalEntries: 0, + totalSize: 0, + oldestEntry: null, + newestEntry: null + }; + } + } +} + +export const localScraperCacheService = LocalScraperCacheService.getInstance(); +export default localScraperCacheService; diff --git a/src/services/localScraperService.ts b/src/services/localScraperService.ts index e1ee92b2..3c49f6c5 100644 --- a/src/services/localScraperService.ts +++ b/src/services/localScraperService.ts @@ -4,6 +4,8 @@ import { Platform } from 'react-native'; import { logger } from '../utils/logger'; import { Stream } from '../types/streams'; import { cacheService } from './cacheService'; +import { localScraperCacheService } from './localScraperCacheService'; +import { hybridCacheService } from './hybridCacheService'; import CryptoJS from 'crypto-js'; // Types for local scrapers @@ -859,7 +861,7 @@ class LocalScraperService { } } - // Execute scrapers for streams + // Execute scrapers for streams with caching async getStreams(type: string, tmdbId: string, season?: number, episode?: number, callback?: ScraperCallback): Promise { await this.ensureInitialized(); @@ -876,20 +878,51 @@ class LocalScraperService { logger.log('[LocalScraperService] No enabled scrapers found for type:', type); return; } + + // Check cache for existing results (hybrid: global first, then local) + const { validResults, expiredScrapers, allExpired, source } = await hybridCacheService.getCachedResults(type, tmdbId, season, episode); - logger.log('[LocalScraperService] Executing', enabledScrapers.length, 'scrapers for', type, tmdbId); + // Immediately return cached results for valid scrapers + if (validResults.length > 0) { + logger.log(`[LocalScraperService] Returning ${validResults.length} cached results for ${type}:${tmdbId} (source: ${source})`); + + for (const cachedResult of validResults) { + if (cachedResult.success && cachedResult.streams.length > 0) { + // Streams are already in the correct format, just pass them through + if (callback) { + callback(cachedResult.streams, cachedResult.scraperId, cachedResult.scraperName, null); + } + } else if (callback) { + // Return error for failed cached results + const error = cachedResult.error ? new Error(cachedResult.error) : new Error('Scraper failed'); + callback(null, cachedResult.scraperId, cachedResult.scraperName, error); + } + } + } + + // Determine which scrapers need to be re-run + const scrapersToRerun = enabledScrapers.filter(scraper => + expiredScrapers.includes(scraper.id) || !validResults.some(r => r.scraperId === scraper.id) + ); + + if (scrapersToRerun.length === 0) { + logger.log('[LocalScraperService] All scrapers have valid cached results'); + return; + } + + logger.log(`[LocalScraperService] Re-running ${scrapersToRerun.length} scrapers (${expiredScrapers.length} expired, ${scrapersToRerun.length - expiredScrapers.length} not cached) for ${type}:${tmdbId}`); // Generate a lightweight request id for tracing const requestId = `rs_${Date.now().toString(36)}_${Math.random().toString(36).slice(2, 6)}`; - // Execute each scraper - for (const scraper of enabledScrapers) { - this.executeScraper(scraper, type, tmdbId, season, episode, callback, requestId); + // Execute only scrapers that need to be re-run + for (const scraper of scrapersToRerun) { + this.executeScraperWithCaching(scraper, type, tmdbId, season, episode, callback, requestId); } } - // Execute individual scraper - private async executeScraper( + // Execute individual scraper with caching + private async executeScraperWithCaching( scraper: ScraperInfo, type: string, tmdbId: string, @@ -904,8 +937,6 @@ class LocalScraperService { throw new Error(`No code found for scraper ${scraper.id}`); } - // Skip verbose logging to reduce CPU load - // Load per-scraper settings const scraperSettings = await this.getScraperSettings(scraper.id); @@ -939,20 +970,57 @@ class LocalScraperService { // Convert results to Nuvio Stream format const streams = this.convertToStreams(results, scraper); + // Cache the successful result (hybrid: both local and global) + await hybridCacheService.cacheScraperResult( + type, + tmdbId, + scraper.id, + scraper.name, + streams, + null, + season, + episode + ); + if (callback) { callback(streams, scraper.id, scraper.name, null); } - // Skip verbose logging to reduce CPU load - } catch (error) { logger.error('[LocalScraperService] Scraper', scraper.name, 'failed:', error); + + // Cache the failed result (hybrid: both local and global) + await hybridCacheService.cacheScraperResult( + type, + tmdbId, + scraper.id, + scraper.name, + null, + error as Error, + season, + episode + ); + if (callback) { callback(null, scraper.id, scraper.name, error as Error); } } } + // Execute individual scraper (legacy method - kept for compatibility) + private async executeScraper( + scraper: ScraperInfo, + type: string, + tmdbId: string, + season?: number, + episode?: number, + callback?: ScraperCallback, + requestId?: string + ): Promise { + // Delegate to the caching version + return this.executeScraperWithCaching(scraper, type, tmdbId, season, episode, callback, requestId); + } + // Execute scraper code in sandboxed environment private async executeSandboxed(code: string, params: any): Promise { // This is a simplified sandbox - in production, you'd want more security @@ -1285,6 +1353,44 @@ class LocalScraperService { await this.ensureInitialized(); return Array.from(this.installedScrapers.values()).some(scraper => scraper.enabled); } + + // Cache management methods (hybrid: local + global) + async clearScraperCache(): Promise { + await hybridCacheService.clearAllCache(); + logger.log('[LocalScraperService] Cleared all scraper cache (local + global)'); + } + + async invalidateScraperCache(scraperId: string): Promise { + await hybridCacheService.invalidateScraper(scraperId); + logger.log('[LocalScraperService] Invalidated cache for scraper:', scraperId); + } + + async invalidateContentCache(type: string, tmdbId: string, season?: number, episode?: number): Promise { + await hybridCacheService.invalidateContent(type, tmdbId, season, episode); + logger.log('[LocalScraperService] Invalidated cache for content:', `${type}:${tmdbId}`); + } + + async getCacheStats(): Promise<{ + local: { + totalEntries: number; + totalSize: number; + oldestEntry: number | null; + newestEntry: number | null; + }; + global: { + totalEntries: number; + totalSize: number; + oldestEntry: number | null; + newestEntry: number | null; + hitRate: number; + }; + combined: { + totalEntries: number; + hitRate: number; + }; + }> { + return await hybridCacheService.getCacheStats(); + } } export const localScraperService = LocalScraperService.getInstance(); diff --git a/src/services/supabaseGlobalCacheService.ts b/src/services/supabaseGlobalCacheService.ts new file mode 100644 index 00000000..a9146847 --- /dev/null +++ b/src/services/supabaseGlobalCacheService.ts @@ -0,0 +1,453 @@ +import { supabase } from './supabaseClient'; +import { logger } from '../utils/logger'; +import { Stream } from '../types/streams'; + +export interface GlobalCachedScraperResult { + streams: Stream[]; + timestamp: number; + success: boolean; + error?: string; + scraperId: string; + scraperName: string; + contentKey: string; // e.g., "movie:123" or "tv:123:1:2" +} + +export interface GlobalCacheStats { + totalEntries: number; + totalSize: number; + oldestEntry: number | null; + newestEntry: number | null; + hitRate: number; +} + +class SupabaseGlobalCacheService { + private static instance: SupabaseGlobalCacheService; + private readonly TABLE_NAME = 'scraper_cache'; + private readonly DEFAULT_TTL_MS = 30 * 60 * 1000; // 30 minutes default TTL + private readonly FAILED_RETRY_TTL_MS = 5 * 60 * 1000; // 5 minutes for failed scrapers + private readonly SUCCESS_TTL_MS = 60 * 60 * 1000; // 1 hour for successful scrapers + private readonly MAX_CACHE_AGE_MS = 7 * 24 * 60 * 60 * 1000; // 7 days max age + private readonly BATCH_SIZE = 50; // Batch size for operations + + // Cache hit/miss tracking + private cacheHits = 0; + private cacheMisses = 0; + + private constructor() {} + + public static getInstance(): SupabaseGlobalCacheService { + if (!SupabaseGlobalCacheService.instance) { + SupabaseGlobalCacheService.instance = new SupabaseGlobalCacheService(); + } + return SupabaseGlobalCacheService.instance; + } + + /** + * Generate cache key for content + */ + private getContentKey(type: string, tmdbId: string, season?: number, episode?: number): string { + if (season !== undefined && episode !== undefined) { + return `${type}:${tmdbId}:${season}:${episode}`; + } + return `${type}:${tmdbId}`; + } + + /** + * Generate unique key for scraper result + */ + private getScraperKey(contentKey: string, scraperId: string): string { + return `${contentKey}:${scraperId}`; + } + + /** + * Check if cached result is still valid based on TTL + */ + private isCacheValid(timestamp: number, success: boolean): boolean { + const ttl = success ? this.SUCCESS_TTL_MS : this.FAILED_RETRY_TTL_MS; + return Date.now() - timestamp < ttl; + } + + /** + * Get cached results for content from global cache + */ + async getCachedResults( + type: string, + tmdbId: string, + season?: number, + episode?: number + ): Promise<{ + validResults: GlobalCachedScraperResult[]; + expiredScrapers: string[]; + allExpired: boolean; + }> { + try { + const contentKey = this.getContentKey(type, tmdbId, season, episode); + + const { data, error } = await supabase + .from(this.TABLE_NAME) + .select('*') + .eq('content_key', contentKey) + .gte('created_at', new Date(Date.now() - this.MAX_CACHE_AGE_MS).toISOString()); + + if (error) { + logger.error('[GlobalCache] Error fetching cached results:', error); + this.cacheMisses++; + return { + validResults: [], + expiredScrapers: [], + allExpired: true + }; + } + + if (!data || data.length === 0) { + this.cacheMisses++; + return { + validResults: [], + expiredScrapers: [], + allExpired: true + }; + } + + // Filter valid results and identify expired scrapers + const validResults: GlobalCachedScraperResult[] = []; + const expiredScrapers: string[] = []; + + for (const row of data) { + const result: GlobalCachedScraperResult = { + streams: row.streams || [], + timestamp: new Date(row.created_at).getTime(), + success: row.success, + error: row.error, + scraperId: row.scraper_id, + scraperName: row.scraper_name, + contentKey: row.content_key + }; + + if (this.isCacheValid(result.timestamp, result.success)) { + validResults.push(result); + } else { + expiredScrapers.push(result.scraperId); + } + } + + // Track cache hits + if (validResults.length > 0) { + this.cacheHits++; + } else { + this.cacheMisses++; + } + + logger.log(`[GlobalCache] Retrieved ${validResults.length} valid results, ${expiredScrapers.length} expired scrapers for ${contentKey}`); + + return { + validResults, + expiredScrapers, + allExpired: validResults.length === 0 + }; + + } catch (error) { + logger.error('[GlobalCache] Error getting cached results:', error); + this.cacheMisses++; + return { + validResults: [], + expiredScrapers: [], + allExpired: true + }; + } + } + + /** + * Cache results for specific scrapers in global cache + */ + async cacheResults( + type: string, + tmdbId: string, + results: Array<{ + scraperId: string; + scraperName: string; + streams: Stream[] | null; + error: Error | null; + }>, + season?: number, + episode?: number + ): Promise { + try { + const contentKey = this.getContentKey(type, tmdbId, season, episode); + const now = new Date().toISOString(); + + // Prepare batch insert data + const insertData = results.map(result => ({ + scraper_key: this.getScraperKey(contentKey, result.scraperId), + content_key: contentKey, + scraper_id: result.scraperId, + scraper_name: result.scraperName, + streams: result.streams || [], + success: !result.error && result.streams !== null, + error: result.error?.message || null, + created_at: now, + updated_at: now + })); + + // Use upsert to handle duplicates + const { error } = await supabase + .from(this.TABLE_NAME) + .upsert(insertData, { + onConflict: 'scraper_key', + ignoreDuplicates: false + }); + + if (error) { + logger.error('[GlobalCache] Error caching results:', error); + } else { + logger.log(`[GlobalCache] Cached ${results.length} results for ${contentKey}`); + } + + } catch (error) { + logger.error('[GlobalCache] Error caching results:', error); + } + } + + /** + * Cache a single scraper result + */ + async cacheScraperResult( + type: string, + tmdbId: string, + scraperId: string, + scraperName: string, + streams: Stream[] | null, + error: Error | null, + season?: number, + episode?: number + ): Promise { + await this.cacheResults(type, tmdbId, [{ + scraperId, + scraperName, + streams, + error + }], season, episode); + } + + /** + * Get list of scrapers that need to be re-run (expired or not cached globally) + */ + async getScrapersToRerun( + type: string, + tmdbId: string, + availableScrapers: Array<{ id: string; name: string }>, + season?: number, + episode?: number + ): Promise { + const { validResults, expiredScrapers } = await this.getCachedResults(type, tmdbId, season, episode); + + const validScraperIds = new Set(validResults.map(r => r.scraperId)); + const expiredScraperIds = new Set(expiredScrapers); + + // Return scrapers that are either expired or not cached globally + const scrapersToRerun = availableScrapers + .filter(scraper => + !validScraperIds.has(scraper.id) || expiredScraperIds.has(scraper.id) + ) + .map(scraper => scraper.id); + + logger.log(`[GlobalCache] Scrapers to re-run: ${scrapersToRerun.join(', ')}`); + + return scrapersToRerun; + } + + /** + * Get all valid cached streams for content from global cache + */ + async getCachedStreams( + type: string, + tmdbId: string, + season?: number, + episode?: number + ): Promise { + const { validResults } = await this.getCachedResults(type, tmdbId, season, episode); + + // Flatten all valid streams + const allStreams: Stream[] = []; + for (const result of validResults) { + if (result.success && result.streams) { + allStreams.push(...result.streams); + } + } + + return allStreams; + } + + /** + * Invalidate cache for specific content globally + */ + async invalidateContent( + type: string, + tmdbId: string, + season?: number, + episode?: number + ): Promise { + try { + const contentKey = this.getContentKey(type, tmdbId, season, episode); + + const { error } = await supabase + .from(this.TABLE_NAME) + .delete() + .eq('content_key', contentKey); + + if (error) { + logger.error('[GlobalCache] Error invalidating cache:', error); + } else { + logger.log(`[GlobalCache] Invalidated global cache for ${contentKey}`); + } + } catch (error) { + logger.error('[GlobalCache] Error invalidating cache:', error); + } + } + + /** + * Invalidate cache for specific scraper across all content globally + */ + async invalidateScraper(scraperId: string): Promise { + try { + const { error } = await supabase + .from(this.TABLE_NAME) + .delete() + .eq('scraper_id', scraperId); + + if (error) { + logger.error('[GlobalCache] Error invalidating scraper cache:', error); + } else { + logger.log(`[GlobalCache] Invalidated global cache for scraper ${scraperId}`); + } + } catch (error) { + logger.error('[GlobalCache] Error invalidating scraper cache:', error); + } + } + + /** + * Clear all cached results globally (admin function) + */ + async clearAllCache(): Promise { + try { + const { error } = await supabase + .from(this.TABLE_NAME) + .delete() + .neq('id', 0); // Delete all rows + + if (error) { + logger.error('[GlobalCache] Error clearing cache:', error); + } else { + logger.log('[GlobalCache] Cleared all global cache'); + } + } catch (error) { + logger.error('[GlobalCache] Error clearing cache:', error); + } + } + + /** + * Clean up old cache entries (older than MAX_CACHE_AGE_MS) + */ + async cleanupOldEntries(): Promise { + try { + const cutoffDate = new Date(Date.now() - this.MAX_CACHE_AGE_MS).toISOString(); + + const { error } = await supabase + .from(this.TABLE_NAME) + .delete() + .lt('created_at', cutoffDate); + + if (error) { + logger.error('[GlobalCache] Error cleaning up old entries:', error); + } else { + logger.log('[GlobalCache] Cleaned up old cache entries'); + } + } catch (error) { + logger.error('[GlobalCache] Error cleaning up old entries:', error); + } + } + + /** + * Get global cache statistics + */ + async getCacheStats(): Promise { + try { + // Get total count + const { count: totalEntries, error: countError } = await supabase + .from(this.TABLE_NAME) + .select('*', { count: 'exact', head: true }); + + if (countError) { + logger.error('[GlobalCache] Error getting cache stats:', countError); + return { + totalEntries: 0, + totalSize: 0, + oldestEntry: null, + newestEntry: null, + hitRate: 0 + }; + } + + // Get oldest and newest entries + const { data: oldestData } = await supabase + .from(this.TABLE_NAME) + .select('created_at') + .order('created_at', { ascending: true }) + .limit(1); + + const { data: newestData } = await supabase + .from(this.TABLE_NAME) + .select('created_at') + .order('created_at', { ascending: false }) + .limit(1); + + const oldestEntry = oldestData?.[0] ? new Date(oldestData[0].created_at).getTime() : null; + const newestEntry = newestData?.[0] ? new Date(newestData[0].created_at).getTime() : null; + + // Calculate hit rate + const totalRequests = this.cacheHits + this.cacheMisses; + const hitRate = totalRequests > 0 ? (this.cacheHits / totalRequests) * 100 : 0; + + return { + totalEntries: totalEntries || 0, + totalSize: 0, // Size calculation would require additional queries + oldestEntry, + newestEntry, + hitRate + }; + } catch (error) { + logger.error('[GlobalCache] Error getting cache stats:', error); + return { + totalEntries: 0, + totalSize: 0, + oldestEntry: null, + newestEntry: null, + hitRate: 0 + }; + } + } + + /** + * Reset cache hit/miss statistics + */ + resetStats(): void { + this.cacheHits = 0; + this.cacheMisses = 0; + } + + /** + * Get cache hit/miss statistics + */ + getHitMissStats(): { hits: number; misses: number; hitRate: number } { + const totalRequests = this.cacheHits + this.cacheMisses; + const hitRate = totalRequests > 0 ? (this.cacheHits / totalRequests) * 100 : 0; + + return { + hits: this.cacheHits, + misses: this.cacheMisses, + hitRate + }; + } +} + +export const supabaseGlobalCacheService = SupabaseGlobalCacheService.getInstance(); +export default supabaseGlobalCacheService;