Creepso/NuvioStreaming
1
0
Fork 0
mirror of https://github.com/tapframe/NuvioStreaming.git synced 2026-03-11 17:45:38 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

multi plugin support

Browse source
This commit is contained in:
tapframe 2025-12-31 19:50:08 +05:30
parent af96d30122
commit 3293b57537
3 changed files with 596 additions and 250 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

232
src/screens/PluginsScreen.tsx
View file

@ -440,6 +440,45 @@ const createStyles = (colors: any) => StyleSheet.create({
color: colors.white, color: colors.white,
fontWeight: '600', fontWeight: '600',
}, },
// Repository tabs
repositoryTabsContainer: {
marginBottom: 16,
},
repositoryTabsScroll: {
flexDirection: 'row',
gap: 8,
},
repositoryTab: {
paddingHorizontal: 16,
paddingVertical: 10,
borderRadius: 10,
backgroundColor: colors.elevation2,
borderWidth: 1,
borderColor: colors.elevation3,
minWidth: 80,
alignItems: 'center',
},
repositoryTabSelected: {
backgroundColor: colors.primary,
borderColor: colors.primary,
},
repositoryTabText: {
color: colors.mediumGray,
fontSize: 14,
fontWeight: '500',
},
repositoryTabTextSelected: {
color: colors.white,
fontWeight: '600',
},
repositoryTabCount: {
fontSize: 12,
color: colors.mediumGray,
marginTop: 2,
},
repositoryTabCountSelected: {
color: 'rgba(255, 255, 255, 0.8)',
},
statusBadge: { statusBadge: {
flexDirection: 'row', flexDirection: 'row',
alignItems: 'center', alignItems: 'center',
@ -761,6 +800,29 @@ const createStyles = (colors: any) => StyleSheet.create({
fontWeight: '500', fontWeight: '500',
color: colors.white, color: colors.white,
}, },
repositoryHeader: {
flexDirection: 'row',
alignItems: 'center',
justifyContent: 'space-between',
marginBottom: 8,
},
repositoryNameContainer: {
flexDirection: 'row',
alignItems: 'center',
flex: 1,
},
pluginRepositoryBadge: {
backgroundColor: colors.elevation3,
paddingHorizontal: 8,
paddingVertical: 2,
borderRadius: 4,
marginRight: 8,
},
pluginRepositoryBadgeText: {
fontSize: 10,
color: colors.mediumGray,
fontWeight: '500',
},
}); });
// Helper component for collapsible sections // Helper component for collapsible sections
@ -881,6 +943,7 @@ const PluginsScreen: React.FC = () => {
// New UX state // New UX state
const [searchQuery, setSearchQuery] = useState(''); const [searchQuery, setSearchQuery] = useState('');
const [selectedFilter, setSelectedFilter] = useState<'all' | 'movie' | 'tv'>('all'); const [selectedFilter, setSelectedFilter] = useState<'all' | 'movie' | 'tv'>('all');
const [selectedRepositoryTab, setSelectedRepositoryTab] = useState<string>('all'); // 'all' or repository ID
const [expandedSections, setExpandedSections] = useState({ const [expandedSections, setExpandedSections] = useState({
repository: true, repository: true,
plugins: true, plugins: true,
@ -904,10 +967,20 @@ const PluginsScreen: React.FC = () => {
{ value: 'SZ', label: 'China' }, { value: 'SZ', label: 'China' },
]; ];
// Filtered plugins based on search and filter // Get enabled repositories for tabs
const enabledRepositories = useMemo(() => {
return repositories.filter(r => r.enabled !== false);
}, [repositories]);
// Filtered plugins based on search, type filter, and repository tab
const filteredPlugins = useMemo(() => { const filteredPlugins = useMemo(() => {
let filtered = installedPlugins; let filtered = installedPlugins;
// Filter by repository tab
if (selectedRepositoryTab !== 'all') {
filtered = filtered.filter(plugin => plugin.repositoryId === selectedRepositoryTab);
}
// Filter by search query // Filter by search query
if (searchQuery.trim()) { if (searchQuery.trim()) {
const query = searchQuery.toLowerCase(); const query = searchQuery.toLowerCase();
@ -926,7 +999,7 @@ const PluginsScreen: React.FC = () => {
} }
return filtered; return filtered;
}, [installedPlugins, searchQuery, selectedFilter]); }, [installedPlugins, searchQuery, selectedFilter, selectedRepositoryTab]);
// Helper functions // Helper functions
const toggleSection = (section: keyof typeof expandedSections) => { const toggleSection = (section: keyof typeof expandedSections) => {
@ -1009,16 +1082,14 @@ const PluginsScreen: React.FC = () => {
enabled: true enabled: true
}); });
await loadRepositories(); // Refresh all enabled repositories to include the new one
await pluginService.refreshRepository();
// Switch to the new repository and refresh it
await pluginService.setCurrentRepository(repoId);
await loadRepositories(); await loadRepositories();
await loadPlugins(); await loadPlugins();
setNewRepositoryUrl(''); setNewRepositoryUrl('');
setShowAddRepositoryModal(false); setShowAddRepositoryModal(false);
openAlert('Success', 'Repository added and refreshed successfully'); openAlert('Success', 'Repository added and plugins loaded successfully');
} catch (error) { } catch (error) {
logger.error('[PluginsScreen] Failed to add repository:', error); logger.error('[PluginsScreen] Failed to add repository:', error);
openAlert('Error', 'Failed to add repository'); openAlert('Error', 'Failed to add repository');
@ -1027,16 +1098,25 @@ const PluginsScreen: React.FC = () => {
} }
}; };
const handleSwitchRepository = async (repoId: string) => { const handleToggleRepositoryEnabled = async (repoId: string, enabled: boolean) => {
try { try {
setSwitchingRepository(repoId); setSwitchingRepository(repoId);
await pluginService.setCurrentRepository(repoId); await pluginService.toggleRepositoryEnabled(repoId, enabled);
if (enabled) {
// When enabling, refresh just this repository to fetch its plugins
await pluginService.refreshSingleRepository(repoId);
}
// Reload the data
await loadRepositories(); await loadRepositories();
await loadPlugins(); await loadPlugins();
openAlert('Success', 'Repository switched successfully');
const repo = repositories.find(r => r.id === repoId);
openAlert('Success', `Repository "${repo?.name || 'Unknown'}" ${enabled ? 'enabled' : 'disabled'} successfully`);
} catch (error) { } catch (error) {
logger.error('[PluginSettings] Failed to switch repository:', error); logger.error('[PluginSettings] Failed to toggle repository:', error);
openAlert('Error', 'Failed to switch repository'); openAlert('Error', 'Failed to update repository');
} finally { } finally {
setSwitchingRepository(null); setSwitchingRepository(null);
} }
@ -1441,40 +1521,43 @@ const PluginsScreen: React.FC = () => {
styles={styles} styles={styles}
> >
<Text style={styles.sectionDescription}> <Text style={styles.sectionDescription}>
Manage multiple plugin repositories. Switch between repositories to access different sets of plugins. Enable multiple repositories to combine plugins from different sources. Toggle each repository on or off below.
</Text> </Text>
{/* Current Repository */}
{currentRepositoryId && (
<View style={styles.currentRepoContainer}>
<Text style={styles.currentRepoLabel}>Current Repository:</Text>
<Text style={styles.currentRepoUrl}>{pluginService.getRepositoryName()}</Text>
<Text style={[styles.currentRepoUrl, { fontSize: 12, opacity: 0.7, marginTop: 4 }]}>{repositoryUrl}</Text>
</View>
)}
{/* Repository List */} {/* Repository List */}
{repositories.length > 0 && ( {repositories.length > 0 && (
<View style={styles.repositoriesList}> <View style={styles.repositoriesList}>
<Text style={[styles.settingTitle, { marginBottom: 12 }]}>Available Repositories</Text> <Text style={[styles.settingTitle, { marginBottom: 8 }]}>Your Repositories</Text>
<Text style={[styles.settingDescription, { marginBottom: 12 }]}>
Enable multiple repositories to combine plugins from different sources.
</Text>
{repositories.map((repo) => ( {repositories.map((repo) => (
<View key={repo.id} style={styles.repositoryItem}> <View key={repo.id} style={[styles.repositoryItem, repo.enabled === false && { opacity: 0.6 }]}>
<View style={styles.repositoryInfo}> <View style={styles.repositoryHeader}>
<View style={{ flexDirection: 'row', alignItems: 'center', marginBottom: 4 }}> <View style={styles.repositoryNameContainer}>
<Text style={styles.repositoryName}>{repo.name}</Text> <Text style={styles.repositoryName}>{repo.name}</Text>
{repo.id === currentRepositoryId && ( {repo.enabled !== false && (
<View style={[styles.statusBadge, { backgroundColor: '#34C759' }]}> <View style={[styles.statusBadge, { backgroundColor: '#34C759' }]}>
<Ionicons name="checkmark-circle" size={12} color="white" /> <Ionicons name="checkmark-circle" size={12} color="white" />
<Text style={styles.statusBadgeText}>Active</Text> <Text style={styles.statusBadgeText}>Enabled</Text>
</View> </View>
)} )}
{switchingRepository === repo.id && ( {switchingRepository === repo.id && (
<View style={[styles.statusBadge, { backgroundColor: colors.primary }]}> <View style={[styles.statusBadge, { backgroundColor: colors.primary }]}>
<ActivityIndicator size={12} color="white" /> <ActivityIndicator size={12} color="white" />
<Text style={styles.statusBadgeText}>Switching...</Text> <Text style={styles.statusBadgeText}>Updating...</Text>
</View> </View>
)} )}
</View> </View>
<Switch
value={repo.enabled !== false}
onValueChange={(enabled) => handleToggleRepositoryEnabled(repo.id, enabled)}
trackColor={{ false: colors.elevation3, true: colors.primary }}
thumbColor={repo.enabled !== false ? colors.white : '#f4f3f4'}
disabled={!settings.enableLocalScrapers || switchingRepository !== null}
/>
</View>
<View style={styles.repositoryInfo}>
{repo.description && ( {repo.description && (
<Text style={styles.repositoryDescription}>{repo.description}</Text> <Text style={styles.repositoryDescription}>{repo.description}</Text>
)} )}
@ -1484,23 +1567,10 @@ const PluginsScreen: React.FC = () => {
</Text> </Text>
</View> </View>
<View style={styles.repositoryActions}> <View style={styles.repositoryActions}>
{repo.id !== currentRepositoryId && (
<TouchableOpacity
style={[styles.repositoryActionButton, styles.repositoryActionButtonPrimary]}
onPress={() => handleSwitchRepository(repo.id)}
disabled={switchingRepository === repo.id}
>
{switchingRepository === repo.id ? (
<ActivityIndicator size="small" color={colors.primary} />
) : (
<Text style={styles.repositoryActionButtonText}>Switch</Text>
)}
</TouchableOpacity>
)}
<TouchableOpacity <TouchableOpacity
style={[styles.repositoryActionButton, styles.repositoryActionButtonSecondary]} style={[styles.repositoryActionButton, styles.repositoryActionButtonSecondary]}
onPress={() => handleRefreshRepository()} onPress={() => handleRefreshRepository()}
disabled={isRefreshing || switchingRepository !== null} disabled={isRefreshing || switchingRepository !== null || repo.enabled === false}
> >
{isRefreshing ? ( {isRefreshing ? (
<ActivityIndicator size="small" color={colors.mediumGray} /> <ActivityIndicator size="small" color={colors.mediumGray} />
@ -1559,6 +1629,70 @@ const PluginsScreen: React.FC = () => {
)} )}
</View> </View>
{/* Repository Tabs - only show if multiple repositories */}
{enabledRepositories.length > 1 && (
<View style={styles.repositoryTabsContainer}>
<ScrollView
horizontal
showsHorizontalScrollIndicator={false}
contentContainerStyle={styles.repositoryTabsScroll}
>
{/* All tab */}
<TouchableOpacity
style={[
styles.repositoryTab,
selectedRepositoryTab === 'all' && styles.repositoryTabSelected
]}
onPress={() => setSelectedRepositoryTab('all')}
>
<Text style={[
styles.repositoryTabText,
selectedRepositoryTab === 'all' && styles.repositoryTabTextSelected
]}>
All
</Text>
<Text style={[
styles.repositoryTabCount,
selectedRepositoryTab === 'all' && styles.repositoryTabCountSelected
]}>
{installedPlugins.length}
</Text>
</TouchableOpacity>
{/* Repository tabs */}
{enabledRepositories.map((repo) => {
const repoPluginCount = installedPlugins.filter(p => p.repositoryId === repo.id).length;
return (
<TouchableOpacity
key={repo.id}
style={[
styles.repositoryTab,
selectedRepositoryTab === repo.id && styles.repositoryTabSelected
]}
onPress={() => setSelectedRepositoryTab(repo.id)}
>
<Text
style={[
styles.repositoryTabText,
selectedRepositoryTab === repo.id && styles.repositoryTabTextSelected
]}
numberOfLines={1}
>
{repo.name}
</Text>
<Text style={[
styles.repositoryTabCount,
selectedRepositoryTab === repo.id && styles.repositoryTabCountSelected
]}>
{repoPluginCount}
</Text>
</TouchableOpacity>
);
})}
</ScrollView>
</View>
)}
{/* Filter Chips */} {/* Filter Chips */}
<View style={styles.filterContainer}> <View style={styles.filterContainer}>
{['all', 'movie', 'tv'].map((filter) => ( {['all', 'movie', 'tv'].map((filter) => (
@ -1574,7 +1708,7 @@ const PluginsScreen: React.FC = () => {
styles.filterChipText, styles.filterChipText,
selectedFilter === filter && styles.filterChipTextSelected selectedFilter === filter && styles.filterChipTextSelected
]}> ]}>
{filter === 'all' ? 'All' : filter === 'movie' ? 'Movies' : 'TV Shows'} {filter === 'all' ? 'All Types' : filter === 'movie' ? 'Movies' : 'TV Shows'}
</Text> </Text>
</TouchableOpacity> </TouchableOpacity>
))} ))}
@ -1693,6 +1827,14 @@ const PluginsScreen: React.FC = () => {
</Text> </Text>
</View> </View>
)} )}
{/* Repository badge */}
{plugin.repositoryId && repositories.length > 1 && (
<View style={styles.pluginRepositoryBadge}>
<Text style={styles.pluginRepositoryBadgeText}>
{repositories.find(r => r.id === plugin.repositoryId)?.name || 'Unknown'}
</Text>
</View>
)}
</View> </View>
{/* ShowBox Settings - only visible when ShowBox plugin is available */} {/* ShowBox Settings - only visible when ShowBox plugin is available */}
@ -1782,7 +1924,7 @@ const PluginsScreen: React.FC = () => {
<View style={styles.settingInfo}> <View style={styles.settingInfo}>
<Text style={styles.settingTitle}>Group Plugin Streams</Text> <Text style={styles.settingTitle}>Group Plugin Streams</Text>
<Text style={styles.settingDescription}> <Text style={styles.settingDescription}>
When enabled, all plugin streams are grouped under "{pluginService.getRepositoryName()}". When disabled, each plugin shows as a separate provider. When enabled, plugin streams are grouped by repository. When disabled, each plugin shows as a separate provider.
</Text> </Text>
</View> </View>
<Switch <Switch

55
src/screens/streams/useStreamsScreen.ts
View file

@ -588,7 +588,11 @@ export const useStreamsScreen = () => {
// Reset provider if no longer available // Reset provider if no longer available
useEffect(() => { useEffect(() => {
const isSpecialFilter = selectedProvider === 'all' || selectedProvider === 'grouped-plugins'; const isSpecialFilter =
selectedProvider === 'all' ||
selectedProvider === 'grouped-plugins' ||
selectedProvider.startsWith('repo-');
if (isSpecialFilter) return; if (isSpecialFilter) return;
const currentStreamsData = selectedEpisode ? episodeStreams : groupedStreams; const currentStreamsData = selectedEpisode ? episodeStreams : groupedStreams;
@ -753,8 +757,23 @@ export const useStreamsScreen = () => {
filterChips.push({ id: provider, name: installedAddon?.name || provider }); filterChips.push({ id: provider, name: installedAddon?.name || provider });
}); });
// Group plugins by repository
if (pluginProviders.length > 0) { if (pluginProviders.length > 0) {
filterChips.push({ id: 'grouped-plugins', name: localScraperService.getRepositoryName() }); const repoMap = new Map<string, { id: string; name: string }>();
pluginProviders.forEach(providerId => {
const repoInfo = localScraperService.getScraperRepository(providerId);
if (repoInfo) {
if (!repoMap.has(repoInfo.id)) {
repoMap.set(repoInfo.id, { id: repoInfo.id, name: repoInfo.name });
}
}
});
// Add a chip for each repository that has plugins with streams
repoMap.forEach(repo => {
filterChips.push({ id: `repo-${repo.id}`, name: repo.name });
});
} }
return filterChips; return filterChips;
@ -789,10 +808,26 @@ export const useStreamsScreen = () => {
const filteredEntries = Object.entries(streams).filter(([addonId]) => { const filteredEntries = Object.entries(streams).filter(([addonId]) => {
if (selectedProvider === 'all') return true; if (selectedProvider === 'all') return true;
// Handle repository-based filtering (repo-{repoId})
if (settings.streamDisplayMode === 'grouped' && selectedProvider && selectedProvider.startsWith('repo-')) {
const repoId = selectedProvider.replace('repo-', '');
if (!repoId) return false;
const isInstalledAddon = installedAddons.some(addon => addon.id === addonId);
if (isInstalledAddon) return false; // Not a plugin
// Check if this plugin belongs to the selected repository
const repoInfo = localScraperService.getScraperRepository(addonId);
return !!(repoInfo && (repoInfo.id === repoId || repoInfo.id?.toLowerCase() === repoId?.toLowerCase()));
}
// Legacy: handle old grouped-plugins filter (fallback)
if (settings.streamDisplayMode === 'grouped' && selectedProvider === 'grouped-plugins') { if (settings.streamDisplayMode === 'grouped' && selectedProvider === 'grouped-plugins') {
const isInstalledAddon = installedAddons.some(addon => addon.id === addonId); const isInstalledAddon = installedAddons.some(addon => addon.id === addonId);
return !isInstalledAddon; return !isInstalledAddon;
} }
return addonId === selectedProvider; return addonId === selectedProvider;
}); });
@ -847,12 +882,24 @@ export const useStreamsScreen = () => {
combinedStreams.push(...pluginStreams); combinedStreams.push(...pluginStreams);
} }
let sectionId = 'grouped-all';
let sectionTitle = 'Available Streams';
if (selectedProvider && selectedProvider.startsWith('repo-')) {
const repoId = selectedProvider.replace('repo-', '');
const repo = localScraperService.getRepository(repoId);
if (repo) {
sectionTitle = `Streams from ${repo.name}`;
sectionId = `grouped-${repoId}`;
}
}
if (combinedStreams.length === 0) return []; if (combinedStreams.length === 0) return [];
return [ return [
{ {
title: 'Available Streams', title: sectionTitle,
addonId: 'grouped-all', addonId: sectionId,
data: combinedStreams, data: combinedStreams,
isEmptyDueToQualityFilter: false, isEmptyDueToQualityFilter: false,
}, },

559
src/services/pluginService.ts
View file

@ -146,21 +146,21 @@ class LocalScraperService {
if (storedScrapers) { if (storedScrapers) {
const scrapers: ScraperInfo[] = JSON.parse(storedScrapers); const scrapers: ScraperInfo[] = JSON.parse(storedScrapers);
const validScrapers: ScraperInfo[] = []; const validScrapers: ScraperInfo[] = [];
scrapers.forEach(scraper => { scrapers.forEach(scraper => {
// Skip scrapers with missing essential fields // Skip scrapers with missing essential fields
if (!scraper.id || !scraper.name || !scraper.version) { if (!scraper.id || !scraper.name || !scraper.version) {
logger.warn('[LocalScraperService] Skipping invalid scraper with missing essential fields:', scraper); logger.warn('[LocalScraperService] Skipping invalid scraper with missing essential fields:', scraper);
return; return;
} }
// Ensure contentLanguage is an array (migration for older scrapers) // Ensure contentLanguage is an array (migration for older scrapers)
if (!scraper.contentLanguage) { if (!scraper.contentLanguage) {
scraper.contentLanguage = ['en']; // Default to English scraper.contentLanguage = ['en']; // Default to English
} else if (typeof scraper.contentLanguage === 'string') { } else if (typeof scraper.contentLanguage === 'string') {
scraper.contentLanguage = [scraper.contentLanguage]; // Convert string to array scraper.contentLanguage = [scraper.contentLanguage]; // Convert string to array
} }
// Ensure supportedTypes is an array (migration for older scrapers) // Ensure supportedTypes is an array (migration for older scrapers)
if (!scraper.supportedTypes || !Array.isArray(scraper.supportedTypes)) { if (!scraper.supportedTypes || !Array.isArray(scraper.supportedTypes)) {
scraper.supportedTypes = ['movie', 'tv']; // Default to both types scraper.supportedTypes = ['movie', 'tv']; // Default to both types
@ -175,7 +175,7 @@ class LocalScraperService {
if (!scraper.supportedFormats && scraper.formats) { if (!scraper.supportedFormats && scraper.formats) {
scraper.supportedFormats = scraper.formats; scraper.supportedFormats = scraper.formats;
} }
// Ensure other required fields have defaults // Ensure other required fields have defaults
if (!scraper.description) { if (!scraper.description) {
scraper.description = 'No description available'; scraper.description = 'No description available';
@ -186,16 +186,16 @@ class LocalScraperService {
if (scraper.enabled === undefined) { if (scraper.enabled === undefined) {
scraper.enabled = true; scraper.enabled = true;
} }
this.installedScrapers.set(scraper.id, scraper); this.installedScrapers.set(scraper.id, scraper);
validScrapers.push(scraper); validScrapers.push(scraper);
}); });
// Save cleaned scrapers back to storage if any were filtered out // Save cleaned scrapers back to storage if any were filtered out
if (validScrapers.length !== scrapers.length) { if (validScrapers.length !== scrapers.length) {
logger.log('[LocalScraperService] Cleaned up invalid scrapers, saving valid ones'); logger.log('[LocalScraperService] Cleaned up invalid scrapers, saving valid ones');
await mmkvStorage.setItem(this.STORAGE_KEY, JSON.stringify(validScrapers)); await mmkvStorage.setItem(this.STORAGE_KEY, JSON.stringify(validScrapers));
// Clean up cached code for removed scrapers // Clean up cached code for removed scrapers
const validScraperIds = new Set(validScrapers.map(s => s.id)); const validScraperIds = new Set(validScrapers.map(s => s.id));
const removedScrapers = scrapers.filter(s => s.id && !validScraperIds.has(s.id)); const removedScrapers = scrapers.filter(s => s.id && !validScraperIds.has(s.id));
@ -212,20 +212,18 @@ class LocalScraperService {
// Load scraper code from cache // Load scraper code from cache
await this.loadScraperCode(); await this.loadScraperCode();
// Auto-refresh repository on app startup if URL is configured (only once) // Auto-refresh ALL enabled repositories on app startup (non-blocking, in background)
if (this.repositoryUrl && !this.autoRefreshCompleted) { const enabledRepos = Array.from(this.repositories.values()).filter(r => r.enabled !== false);
try { if (enabledRepos.length > 0 && !this.autoRefreshCompleted) {
logger.log('[LocalScraperService] Auto-refreshing repository on startup'); this.autoRefreshCompleted = true; // Mark immediately to prevent duplicate calls
await this.performRepositoryRefresh(); logger.log('[LocalScraperService] Scheduling background refresh of', enabledRepos.length, 'enabled repositories');
this.autoRefreshCompleted = true; // Don't await - let it run in background so app loads fast
} catch (error) { this.refreshAllEnabledRepositories().catch(error => {
logger.error('[LocalScraperService] Auto-refresh failed on startup:', error); logger.error('[LocalScraperService] Background auto-refresh failed:', error);
// Don't fail initialization if auto-refresh fails });
this.autoRefreshCompleted = true; // Mark as completed even on error to prevent retries
}
} }
this.initialized = true; this.initialized = true;
logger.log('[LocalScraperService] Initialized with', this.installedScrapers.size, 'scrapers'); logger.log('[LocalScraperService] Initialized with', this.installedScrapers.size, 'scrapers');
} catch (error) { } catch (error) {
@ -268,7 +266,7 @@ class LocalScraperService {
async addRepository(repo: Omit<RepositoryInfo, 'id' | 'lastUpdated' | 'scraperCount'>): Promise<string> { async addRepository(repo: Omit<RepositoryInfo, 'id' | 'lastUpdated' | 'scraperCount'>): Promise<string> {
await this.ensureInitialized(); await this.ensureInitialized();
const id = `repo_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`; const id = `repo_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`;
// Try to fetch the repository name from manifest if not provided // Try to fetch the repository name from manifest if not provided
let repositoryName = repo.name; let repositoryName = repo.name;
if (!repositoryName || repositoryName.trim() === '') { if (!repositoryName || repositoryName.trim() === '') {
@ -279,7 +277,7 @@ class LocalScraperService {
repositoryName = this.extractRepositoryName(repo.url); repositoryName = this.extractRepositoryName(repo.url);
} }
} }
const newRepo: RepositoryInfo = { const newRepo: RepositoryInfo = {
...repo, ...repo,
name: repositoryName, name: repositoryName,
@ -302,7 +300,7 @@ class LocalScraperService {
const updatedRepo = { ...repo, ...updates }; const updatedRepo = { ...repo, ...updates };
this.repositories.set(id, updatedRepo); this.repositories.set(id, updatedRepo);
await this.saveRepositories(); await this.saveRepositories();
// If this is the current repository, update current values // If this is the current repository, update current values
if (id === this.currentRepositoryId) { if (id === this.currentRepositoryId) {
this.repositoryUrl = updatedRepo.url; this.repositoryUrl = updatedRepo.url;
@ -316,10 +314,10 @@ class LocalScraperService {
if (!this.repositories.has(id)) { if (!this.repositories.has(id)) {
throw new Error(`Repository with id ${id} not found`); throw new Error(`Repository with id ${id} not found`);
} }
// Allow removing the last repository - users can add new ones // Allow removing the last repository - users can add new ones
// The app will work without repositories (no scrapers available) // The app will work without repositories (no scrapers available)
// If removing current repository, switch to another one or clear current // If removing current repository, switch to another one or clear current
if (id === this.currentRepositoryId) { if (id === this.currentRepositoryId) {
const remainingRepos = Array.from(this.repositories.values()).filter(r => r.id !== id); const remainingRepos = Array.from(this.repositories.values()).filter(r => r.id !== id);
@ -331,18 +329,18 @@ class LocalScraperService {
await mmkvStorage.removeItem('current-repository-id'); await mmkvStorage.removeItem('current-repository-id');
} }
} }
// Remove scrapers from this repository // Remove scrapers from this repository
const scrapersToRemove = Array.from(this.installedScrapers.values()) const scrapersToRemove = Array.from(this.installedScrapers.values())
.filter(s => s.repositoryId === id) .filter(s => s.repositoryId === id)
.map(s => s.id); .map(s => s.id);
for (const scraperId of scrapersToRemove) { for (const scraperId of scrapersToRemove) {
this.installedScrapers.delete(scraperId); this.installedScrapers.delete(scraperId);
this.scraperCode.delete(scraperId); this.scraperCode.delete(scraperId);
await mmkvStorage.removeItem(`scraper-code-${scraperId}`); await mmkvStorage.removeItem(`scraper-code-${scraperId}`);
} }
this.repositories.delete(id); this.repositories.delete(id);
await this.saveRepositories(); await this.saveRepositories();
await this.saveInstalledScrapers(); await this.saveInstalledScrapers();
@ -355,13 +353,13 @@ class LocalScraperService {
if (!repo) { if (!repo) {
throw new Error(`Repository with id ${id} not found`); throw new Error(`Repository with id ${id} not found`);
} }
this.currentRepositoryId = id; this.currentRepositoryId = id;
this.repositoryUrl = repo.url; this.repositoryUrl = repo.url;
this.repositoryName = repo.name; this.repositoryName = repo.name;
await mmkvStorage.setItem('current-repository-id', id); await mmkvStorage.setItem('current-repository-id', id);
// Refresh the repository to get its scrapers // Refresh the repository to get its scrapers
try { try {
logger.log('[LocalScraperService] Refreshing repository after switch:', repo.name); logger.log('[LocalScraperService] Refreshing repository after switch:', repo.name);
@ -370,7 +368,7 @@ class LocalScraperService {
logger.error('[LocalScraperService] Failed to refresh repository after switch:', error); logger.error('[LocalScraperService] Failed to refresh repository after switch:', error);
// Don't throw error, just log it - the switch should still succeed // Don't throw error, just log it - the switch should still succeed
} }
logger.log('[LocalScraperService] Switched to repository:', repo.name); logger.log('[LocalScraperService] Switched to repository:', repo.name);
} }
@ -396,21 +394,21 @@ class LocalScraperService {
async fetchRepositoryNameFromManifest(repositoryUrl: string): Promise<string> { async fetchRepositoryNameFromManifest(repositoryUrl: string): Promise<string> {
try { try {
logger.log('[LocalScraperService] Fetching repository name from manifest:', repositoryUrl); logger.log('[LocalScraperService] Fetching repository name from manifest:', repositoryUrl);
// Construct manifest URL // Construct manifest URL
const baseManifestUrl = repositoryUrl.endsWith('/') const baseManifestUrl = repositoryUrl.endsWith('/')
? `${repositoryUrl}manifest.json` ? `${repositoryUrl}manifest.json`
: `${repositoryUrl}/manifest.json`; : `${repositoryUrl}/manifest.json`;
const manifestUrl = `${baseManifestUrl}?t=${Date.now()}`; const manifestUrl = `${baseManifestUrl}?t=${Date.now()}`;
const response = await axios.get(manifestUrl, { const response = await axios.get(manifestUrl, {
timeout: 10000, timeout: 10000,
headers: { headers: {
'Cache-Control': 'no-cache', 'Cache-Control': 'no-cache',
'Pragma': 'no-cache' 'Pragma': 'no-cache'
} }
}); });
if (response.data && response.data.name) { if (response.data && response.data.name) {
logger.log('[LocalScraperService] Found repository name in manifest:', response.data.name); logger.log('[LocalScraperService] Found repository name in manifest:', response.data.name);
return response.data.name; return response.data.name;
@ -427,14 +425,14 @@ class LocalScraperService {
// Update repository name from manifest for existing repositories // Update repository name from manifest for existing repositories
async refreshRepositoryNamesFromManifests(): Promise<void> { async refreshRepositoryNamesFromManifests(): Promise<void> {
await this.ensureInitialized(); await this.ensureInitialized();
for (const [id, repo] of this.repositories) { for (const [id, repo] of this.repositories) {
try { try {
const manifestName = await this.fetchRepositoryNameFromManifest(repo.url); const manifestName = await this.fetchRepositoryNameFromManifest(repo.url);
if (manifestName !== repo.name) { if (manifestName !== repo.name) {
logger.log('[LocalScraperService] Updating repository name:', repo.name, '->', manifestName); logger.log('[LocalScraperService] Updating repository name:', repo.name, '->', manifestName);
repo.name = manifestName; repo.name = manifestName;
// If this is the current repository, update the current name // If this is the current repository, update the current name
if (id === this.currentRepositoryId) { if (id === this.currentRepositoryId) {
this.repositoryName = manifestName; this.repositoryName = manifestName;
@ -444,7 +442,7 @@ class LocalScraperService {
logger.warn('[LocalScraperService] Failed to refresh name for repository:', repo.name, error); logger.warn('[LocalScraperService] Failed to refresh name for repository:', repo.name, error);
} }
} }
await this.saveRepositories(); await this.saveRepositories();
} }
@ -453,23 +451,56 @@ class LocalScraperService {
await mmkvStorage.setItem(this.REPOSITORIES_KEY, JSON.stringify(reposObject)); await mmkvStorage.setItem(this.REPOSITORIES_KEY, JSON.stringify(reposObject));
} }
// Get a single repository
getRepository(id: string): RepositoryInfo | undefined {
return this.repositories.get(id);
}
// Get all enabled repositories (for multi-repo support)
async getEnabledRepositories(): Promise<RepositoryInfo[]> {
await this.ensureInitialized();
return Array.from(this.repositories.values()).filter(r => r.enabled !== false);
}
// Toggle a repository's enabled state (for multi-repo support)
async toggleRepositoryEnabled(id: string, enabled: boolean): Promise<void> {
await this.ensureInitialized();
const repo = this.repositories.get(id);
if (!repo) {
throw new Error(`Repository with id ${id} not found`);
}
repo.enabled = enabled;
this.repositories.set(id, repo);
await this.saveRepositories();
logger.log('[LocalScraperService] Toggled repository', repo.name, 'to', enabled ? 'enabled' : 'disabled');
}
// Get the repository info for a scraper
getScraperRepository(scraperId: string): RepositoryInfo | undefined {
const scraper = this.installedScrapers.get(scraperId);
if (!scraper?.repositoryId) return undefined;
return this.repositories.get(scraper.repositoryId);
}
// Check if a scraper is compatible with the current platform // Check if a scraper is compatible with the current platform
private isPlatformCompatible(scraper: ScraperInfo): boolean { private isPlatformCompatible(scraper: ScraperInfo): boolean {
const currentPlatform = Platform.OS as 'ios' | 'android'; const currentPlatform = Platform.OS as 'ios' | 'android';
// If disabledPlatforms is specified and includes current platform, scraper is not compatible // If disabledPlatforms is specified and includes current platform, scraper is not compatible
if (scraper.disabledPlatforms && scraper.disabledPlatforms.includes(currentPlatform)) { if (scraper.disabledPlatforms && scraper.disabledPlatforms.includes(currentPlatform)) {
logger.log(`[LocalScraperService] Scraper ${scraper.name} is disabled on ${currentPlatform}`); logger.log(`[LocalScraperService] Scraper ${scraper.name} is disabled on ${currentPlatform}`);
return false; return false;
} }
// If supportedPlatforms is specified and doesn't include current platform, scraper is not compatible // If supportedPlatforms is specified and doesn't include current platform, scraper is not compatible
if (scraper.supportedPlatforms && !scraper.supportedPlatforms.includes(currentPlatform)) { if (scraper.supportedPlatforms && !scraper.supportedPlatforms.includes(currentPlatform)) {
logger.log(`[LocalScraperService] Scraper ${scraper.name} is not supported on ${currentPlatform}`); logger.log(`[LocalScraperService] Scraper ${scraper.name} is not supported on ${currentPlatform}`);
return false; return false;
} }
// If neither supportedPlatforms nor disabledPlatforms is specified, or current platform is supported // If neither supportedPlatforms nor disabledPlatforms is specified, or current platform is supported
return true; return true;
} }
@ -477,10 +508,103 @@ class LocalScraperService {
// Fetch and install scrapers from repository // Fetch and install scrapers from repository
async refreshRepository(): Promise<void> { async refreshRepository(): Promise<void> {
await this.ensureInitialized(); await this.ensureInitialized();
await this.performRepositoryRefresh(); await this.refreshAllEnabledRepositories();
this.autoRefreshCompleted = true; // Mark as completed after manual refresh this.autoRefreshCompleted = true; // Mark as completed after manual refresh
} }
// Refresh ALL enabled repositories (for multi-repo support)
async refreshAllEnabledRepositories(): Promise<void> {
await this.ensureInitialized();
const enabledRepos = await this.getEnabledRepositories();
if (enabledRepos.length === 0) {
logger.log('[LocalScraperService] No enabled repositories to refresh');
// Clear all caches when no repositories are enabled
this.scraperCode.clear();
this.installedScrapers.clear();
this.inFlightByKey.clear();
this.scraperSettingsCache = null;
try {
cacheService.clearCache();
} catch (error) {
logger.warn('[LocalScraperService] Failed to clear cacheService:', error);
}
await this.saveInstalledScrapers();
return;
}
logger.log('[LocalScraperService] Refreshing', enabledRepos.length, 'enabled repositories...');
// IMPORTANT: Preserve user's enabled preferences before clearing
const previousEnabledStates = new Map<string, boolean>();
for (const [id, scraper] of this.installedScrapers) {
previousEnabledStates.set(id, scraper.enabled);
}
// Store it on the instance so downloadScraper can access it
(this as any)._previousEnabledStates = previousEnabledStates;
// Clear caches before refreshing all
this.scraperCode.clear();
this.installedScrapers.clear();
this.inFlightByKey.clear();
this.scraperSettingsCache = null;
try {
const allKeys = await mmkvStorage.getAllKeys();
const scraperCodeKeys = allKeys.filter(key => key.startsWith('scraper-code-'));
if (scraperCodeKeys.length > 0) {
await mmkvStorage.multiRemove(scraperCodeKeys);
logger.log('[LocalScraperService] Removed', scraperCodeKeys.length, 'cached scraper code entries');
}
} catch (error) {
logger.error('[LocalScraperService] Failed to clear cached scraper code:', error);
}
try {
cacheService.clearCache();
logger.log('[LocalScraperService] Cleared cacheService during refresh');
} catch (error) {
logger.warn('[LocalScraperService] Failed to clear cacheService:', error);
}
// Refresh all enabled repositories in PARALLEL for faster loading
logger.log('[LocalScraperService] Starting parallel refresh of', enabledRepos.length, 'repositories...');
const refreshResults = await Promise.allSettled(
enabledRepos.map(repo => this.refreshSingleRepository(repo.id))
);
// Log results
refreshResults.forEach((result, index) => {
const repo = enabledRepos[index];
if (result.status === 'fulfilled') {
logger.log('[LocalScraperService] Successfully refreshed repository:', repo.name);
} else {
logger.error('[LocalScraperService] Failed to refresh repository:', repo.name, result.reason);
}
});
await this.saveInstalledScrapers();
// Clean up the temporary preserved states
delete (this as any)._previousEnabledStates;
logger.log('[LocalScraperService] Finished refreshing all enabled repositories. Total scrapers:', this.installedScrapers.size);
}
// Refresh a single repository by ID (parallel-safe - no shared state mutation)
async refreshSingleRepository(repoId: string): Promise<void> {
await this.ensureInitialized();
const repo = this.repositories.get(repoId);
if (!repo) {
throw new Error(`Repository with id ${repoId} not found`);
}
// Directly call performSingleRepositoryRefresh - it handles everything with explicit repo object
await this.performSingleRepositoryRefresh(repo);
}
// Internal method to refresh repository without initialization check // Internal method to refresh repository without initialization check
private async performRepositoryRefresh(): Promise<void> { private async performRepositoryRefresh(): Promise<void> {
if (!this.repositoryUrl) { if (!this.repositoryUrl) {
@ -516,33 +640,33 @@ class LocalScraperService {
const manifestUrl = `${baseManifestUrl}?t=${Date.now()}&v=${Math.random()}`; const manifestUrl = `${baseManifestUrl}?t=${Date.now()}&v=${Math.random()}`;
const response = await axios.get(manifestUrl, { const response = await axios.get(manifestUrl, {
timeout: 10000, timeout: 10000,
headers: { headers: {
'Cache-Control': 'no-cache', 'Cache-Control': 'no-cache',
'Pragma': 'no-cache', 'Pragma': 'no-cache',
'Expires': '0' 'Expires': '0'
} }
}); });
const manifest: ScraperManifest = response.data; const manifest: ScraperManifest = response.data;
// Store repository name from manifest // Store repository name from manifest
if (manifest.name) { if (manifest.name) {
this.repositoryName = manifest.name; this.repositoryName = manifest.name;
} }
logger.log('[LocalScraperService] getAvailableScrapers - Raw manifest data:', JSON.stringify(manifest, null, 2)); logger.log('[LocalScraperService] getAvailableScrapers - Raw manifest data:', JSON.stringify(manifest, null, 2));
logger.log('[LocalScraperService] getAvailableScrapers - Manifest scrapers count:', manifest.scrapers?.length || 0); logger.log('[LocalScraperService] getAvailableScrapers - Manifest scrapers count:', manifest.scrapers?.length || 0);
// Log each scraper's enabled status from manifest // Log each scraper's enabled status from manifest
manifest.scrapers?.forEach(scraper => { manifest.scrapers?.forEach(scraper => {
logger.log(`[LocalScraperService] getAvailableScrapers - Scraper ${scraper.name}: enabled=${scraper.enabled}`); logger.log(`[LocalScraperService] getAvailableScrapers - Scraper ${scraper.name}: enabled=${scraper.enabled}`);
}); });
logger.log('[LocalScraperService] Found', manifest.scrapers.length, 'scrapers in repository'); logger.log('[LocalScraperService] Found', manifest.scrapers.length, 'scrapers in repository');
// Get current manifest scraper IDs // Get current manifest scraper IDs
const manifestScraperIds = new Set(manifest.scrapers.map(s => s.id)); const manifestScraperIds = new Set(manifest.scrapers.map(s => s.id));
// Remove scrapers that are no longer in the manifest // Remove scrapers that are no longer in the manifest
const currentScraperIds = Array.from(this.installedScrapers.keys()); const currentScraperIds = Array.from(this.installedScrapers.keys());
for (const scraperId of currentScraperIds) { for (const scraperId of currentScraperIds) {
@ -554,11 +678,11 @@ class LocalScraperService {
await mmkvStorage.removeItem(`scraper-code-${scraperId}`); await mmkvStorage.removeItem(`scraper-code-${scraperId}`);
} }
} }
// Download and install each scraper from manifest // Download and install each scraper from manifest
for (const scraperInfo of manifest.scrapers) { for (const scraperInfo of manifest.scrapers) {
const isPlatformCompatible = this.isPlatformCompatible(scraperInfo); const isPlatformCompatible = this.isPlatformCompatible(scraperInfo);
if (isPlatformCompatible) { if (isPlatformCompatible) {
// Add repository ID to scraper info // Add repository ID to scraper info
const scraperWithRepo = { ...scraperInfo, repositoryId: this.currentRepositoryId }; const scraperWithRepo = { ...scraperInfo, repositoryId: this.currentRepositoryId };
@ -575,9 +699,9 @@ class LocalScraperService {
} }
} }
} }
await this.saveInstalledScrapers(); await this.saveInstalledScrapers();
// Update repository info // Update repository info
const currentRepo = this.repositories.get(this.currentRepositoryId); const currentRepo = this.repositories.get(this.currentRepositoryId);
if (currentRepo) { if (currentRepo) {
@ -588,9 +712,9 @@ class LocalScraperService {
scraperCount scraperCount
}); });
} }
logger.log('[LocalScraperService] Repository refresh completed'); logger.log('[LocalScraperService] Repository refresh completed');
} catch (error) { } catch (error) {
logger.error('[LocalScraperService] Failed to refresh repository:', error); logger.error('[LocalScraperService] Failed to refresh repository:', error);
throw error; throw error;
@ -599,15 +723,97 @@ class LocalScraperService {
} }
} }
// Download individual scraper // Refresh a single repository without clearing others (for multi-repo support)
private async downloadScraper(scraperInfo: ScraperInfo): Promise<void> { private async performSingleRepositoryRefresh(repo: RepositoryInfo): Promise<void> {
logger.log('[LocalScraperService] Fetching repository manifest from:', repo.url);
try { try {
const scraperUrl = this.repositoryUrl.endsWith('/') // Fetch manifest with cache busting
? `${this.repositoryUrl}${scraperInfo.filename}` const baseManifestUrl = repo.url.endsWith('/')
: `${this.repositoryUrl}/${scraperInfo.filename}`; ? `${repo.url}manifest.json`
: `${repo.url}/manifest.json`;
const manifestUrl = `${baseManifestUrl}?t=${Date.now()}&v=${Math.random()}`;
const response = await axios.get(manifestUrl, {
timeout: 10000,
headers: {
'Cache-Control': 'no-cache',
'Pragma': 'no-cache',
'Expires': '0'
}
});
const manifest: ScraperManifest = response.data;
// Update repository name from manifest
if (manifest.name) {
repo.name = manifest.name;
this.repositories.set(repo.id, repo);
}
logger.log('[LocalScraperService] Repository', repo.name, 'has', manifest.scrapers?.length || 0, 'scrapers');
// Get current manifest scraper IDs for this repository
const manifestScraperIds = new Set(manifest.scrapers.map(s => s.id));
// Remove scrapers from this repository that are no longer in its manifest
const currentScraperIds = Array.from(this.installedScrapers.keys());
for (const scraperId of currentScraperIds) {
const scraper = this.installedScrapers.get(scraperId);
if (scraper?.repositoryId === repo.id && !manifestScraperIds.has(scraperId)) {
logger.log('[LocalScraperService] Removing scraper no longer in manifest:', scraper.name);
this.installedScrapers.delete(scraperId);
this.scraperCode.delete(scraperId);
await mmkvStorage.removeItem(`scraper-code-${scraperId}`);
}
}
// Download and install each scraper from manifest
for (const scraperInfo of manifest.scrapers) {
const isPlatformCompatible = this.isPlatformCompatible(scraperInfo);
if (isPlatformCompatible) {
// Add repository ID to scraper info
const scraperWithRepo = { ...scraperInfo, repositoryId: repo.id };
// Download/update the scraper - pass repo.url explicitly for parallel-safe operation
await this.downloadScraper(scraperWithRepo, repo.url);
} else {
logger.log('[LocalScraperService] Skipping platform-incompatible scraper:', scraperInfo.name);
// Remove if it was previously installed but is now platform-incompatible
if (this.installedScrapers.has(scraperInfo.id)) {
this.installedScrapers.delete(scraperInfo.id);
this.scraperCode.delete(scraperInfo.id);
await mmkvStorage.removeItem(`scraper-code-${scraperInfo.id}`);
}
}
}
// Update repository info
const scraperCount = Array.from(this.installedScrapers.values())
.filter(s => s.repositoryId === repo.id).length;
await this.updateRepository(repo.id, {
lastUpdated: Date.now(),
scraperCount
});
logger.log('[LocalScraperService] Repository', repo.name, 'refresh completed with', scraperCount, 'scrapers');
} catch (error) {
logger.error('[LocalScraperService] Failed to refresh repository:', repo.name, error);
throw error;
}
}
// Download individual scraper (repositoryUrl passed explicitly for parallel-safe operation)
private async downloadScraper(scraperInfo: ScraperInfo, repositoryUrl?: string): Promise<void> {
try {
// Use passed repositoryUrl or fall back to this.repositoryUrl for backward compatibility
const repoUrl = repositoryUrl || this.repositoryUrl;
const scraperUrl = repoUrl.endsWith('/')
? `${repoUrl}${scraperInfo.filename}`
: `${repoUrl}/${scraperInfo.filename}`;
logger.log('[LocalScraperService] Downloading scraper:', scraperInfo.name); logger.log('[LocalScraperService] Downloading scraper:', scraperInfo.name);
// Add cache-busting parameters to force fresh download // Add cache-busting parameters to force fresh download
const scraperUrlWithCacheBust = `${scraperUrl}?t=${Date.now()}&v=${Math.random()}`; const scraperUrlWithCacheBust = `${scraperUrl}?t=${Date.now()}&v=${Math.random()}`;
@ -620,11 +826,16 @@ class LocalScraperService {
} }
}); });
const scraperCode = response.data; const scraperCode = response.data;
// Store scraper info and code // Store scraper info and code
const existingScraper = this.installedScrapers.get(scraperInfo.id); const existingScraper = this.installedScrapers.get(scraperInfo.id);
const isPlatformCompatible = this.isPlatformCompatible(scraperInfo); const isPlatformCompatible = this.isPlatformCompatible(scraperInfo);
// Check preserved states first (from refresh), then existing scraper, then default
const previousStates = (this as any)._previousEnabledStates as Map<string, boolean> | undefined;
const previousEnabled = previousStates?.get(scraperInfo.id);
const userEnabledState = previousEnabled !== undefined ? previousEnabled : (existingScraper?.enabled ?? true);
const updatedScraperInfo = { const updatedScraperInfo = {
...scraperInfo, ...scraperInfo,
// Store the manifest's enabled state separately // Store the manifest's enabled state separately
@ -632,17 +843,17 @@ class LocalScraperService {
// Force disable if: // Force disable if:
// 1. Manifest says enabled: false (globally disabled) // 1. Manifest says enabled: false (globally disabled)
// 2. Platform incompatible // 2. Platform incompatible
// Otherwise, preserve user's enabled state or default to true for new installations // Otherwise, preserve user's enabled state
enabled: scraperInfo.enabled && isPlatformCompatible ? (existingScraper?.enabled ?? true) : false enabled: scraperInfo.enabled && isPlatformCompatible ? userEnabledState : false
}; };
// Ensure contentLanguage is an array (migration for older scrapers) // Ensure contentLanguage is an array (migration for older scrapers)
if (!updatedScraperInfo.contentLanguage) { if (!updatedScraperInfo.contentLanguage) {
updatedScraperInfo.contentLanguage = ['en']; // Default to English updatedScraperInfo.contentLanguage = ['en']; // Default to English
} else if (typeof updatedScraperInfo.contentLanguage === 'string') { } else if (typeof updatedScraperInfo.contentLanguage === 'string') {
updatedScraperInfo.contentLanguage = [updatedScraperInfo.contentLanguage]; // Convert string to array updatedScraperInfo.contentLanguage = [updatedScraperInfo.contentLanguage]; // Convert string to array
} }
// Ensure supportedTypes is an array (migration for older scrapers) // Ensure supportedTypes is an array (migration for older scrapers)
if (!updatedScraperInfo.supportedTypes || !Array.isArray(updatedScraperInfo.supportedTypes)) { if (!updatedScraperInfo.supportedTypes || !Array.isArray(updatedScraperInfo.supportedTypes)) {
updatedScraperInfo.supportedTypes = ['movie', 'tv']; // Default to both types updatedScraperInfo.supportedTypes = ['movie', 'tv']; // Default to both types
@ -657,16 +868,16 @@ class LocalScraperService {
if (!updatedScraperInfo.supportedFormats && updatedScraperInfo.formats) { if (!updatedScraperInfo.supportedFormats && updatedScraperInfo.formats) {
updatedScraperInfo.supportedFormats = updatedScraperInfo.formats; updatedScraperInfo.supportedFormats = updatedScraperInfo.formats;
} }
this.installedScrapers.set(scraperInfo.id, updatedScraperInfo); this.installedScrapers.set(scraperInfo.id, updatedScraperInfo);
this.scraperCode.set(scraperInfo.id, scraperCode); this.scraperCode.set(scraperInfo.id, scraperCode);
// Cache the scraper code // Cache the scraper code
await this.cacheScraperCode(scraperInfo.id, scraperCode); await this.cacheScraperCode(scraperInfo.id, scraperCode);
logger.log('[LocalScraperService] Successfully downloaded:', scraperInfo.name); logger.log('[LocalScraperService] Successfully downloaded:', scraperInfo.name);
} catch (error) { } catch (error) {
logger.error('[LocalScraperService] Failed to download scraper', scraperInfo.name, ':', error); logger.error('[LocalScraperService] Failed to download scraper', scraperInfo.name, ':', error);
} }
@ -743,83 +954,29 @@ class LocalScraperService {
} }
} }
// Get available scrapers from manifest.json (for display in settings) // Get available scrapers from ALL enabled repositories (for display in settings)
async getAvailableScrapers(): Promise<ScraperInfo[]> { async getAvailableScrapers(): Promise<ScraperInfo[]> {
if (!this.repositoryUrl) { await this.ensureInitialized();
logger.log('[LocalScraperService] No repository URL configured, returning installed scrapers');
return this.getInstalledScrapers(); const enabledRepos = await this.getEnabledRepositories();
if (enabledRepos.length === 0) {
logger.log('[LocalScraperService] No enabled repositories, returning empty list');
return [];
} }
try { // Return installed scrapers from all enabled repositories
logger.log('[LocalScraperService] Fetching available scrapers from manifest'); // These are already synced with manifests during refresh
const allScrapers = Array.from(this.installedScrapers.values())
// Fetch manifest with cache busting .filter(scraper => {
const baseManifestUrl = this.repositoryUrl.endsWith('/') // Only include scrapers from enabled repositories
? `${this.repositoryUrl}manifest.json` const repo = this.repositories.get(scraper.repositoryId || '');
: `${this.repositoryUrl}/manifest.json`; return repo?.enabled !== false;
const manifestUrl = `${baseManifestUrl}?t=${Date.now()}&v=${Math.random()}`; });
const response = await axios.get(manifestUrl, { logger.log('[LocalScraperService] Found', allScrapers.length, 'scrapers from', enabledRepos.length, 'enabled repositories');
timeout: 10000,
headers: {
'Cache-Control': 'no-cache',
'Pragma': 'no-cache',
'Expires': '0'
}
});
const manifest: ScraperManifest = response.data;
// Store repository name from manifest
if (manifest.name) {
this.repositoryName = manifest.name;
}
// Return scrapers from manifest, respecting manifest's enabled field and platform compatibility
const availableScrapers = manifest.scrapers
.filter(scraperInfo => this.isPlatformCompatible(scraperInfo))
.map(scraperInfo => {
const installedScraper = this.installedScrapers.get(scraperInfo.id);
// Create a copy with manifest data
const scraperWithManifestData = {
...scraperInfo,
// Store the manifest's enabled state separately
manifestEnabled: scraperInfo.enabled,
// If manifest says enabled: false, scraper cannot be enabled
// If manifest says enabled: true, use installed state or default to true for new installs
enabled: scraperInfo.enabled ? (installedScraper?.enabled ?? true) : false
};
return allScrapers;
// Normalize formats fields (support both `formats` and `supportedFormats`)
const anyScraper: any = scraperWithManifestData as any;
if (typeof anyScraper.formats === 'string') {
anyScraper.formats = [anyScraper.formats];
}
if (typeof anyScraper.supportedFormats === 'string') {
anyScraper.supportedFormats = [anyScraper.supportedFormats];
}
if (!anyScraper.supportedFormats && anyScraper.formats) {
anyScraper.supportedFormats = anyScraper.formats;
}
return scraperWithManifestData;
});
logger.log('[LocalScraperService] Found', availableScrapers.length, 'available scrapers in repository');
// Log final scraper states being returned to UI
availableScrapers.forEach(scraper => {
logger.log(`[LocalScraperService] Final scraper ${scraper.name}: manifestEnabled=${scraper.manifestEnabled}, enabled=${scraper.enabled}`);
});
return availableScrapers;
} catch (error) {
logger.error('[LocalScraperService] Failed to fetch available scrapers from manifest:', error);
// Fallback to installed scrapers if manifest fetch fails
return this.getInstalledScrapers();
}
} }
// Check if a given scraper declares support for a specific format (e.g., 'mkv') // Check if a given scraper declares support for a specific format (e.g., 'mkv')
@ -844,7 +1001,7 @@ class LocalScraperService {
// Enable/disable scraper // Enable/disable scraper
async setScraperEnabled(scraperId: string, enabled: boolean): Promise<void> { async setScraperEnabled(scraperId: string, enabled: boolean): Promise<void> {
await this.ensureInitialized(); await this.ensureInitialized();
const scraper = this.installedScrapers.get(scraperId); const scraper = this.installedScrapers.get(scraperId);
if (scraper) { if (scraper) {
// Prevent enabling if manifest has disabled it or if platform-incompatible // Prevent enabling if manifest has disabled it or if platform-incompatible
@ -852,7 +1009,7 @@ class LocalScraperService {
logger.log('[LocalScraperService] Cannot enable scraper', scraperId, '- disabled in manifest or platform-incompatible'); logger.log('[LocalScraperService] Cannot enable scraper', scraperId, '- disabled in manifest or platform-incompatible');
return; return;
} }
scraper.enabled = enabled; scraper.enabled = enabled;
this.installedScrapers.set(scraperId, scraper); this.installedScrapers.set(scraperId, scraper);
await this.saveInstalledScrapers(); await this.saveInstalledScrapers();
@ -920,7 +1077,7 @@ class LocalScraperService {
if (enabledScrapers.length > 0) { if (enabledScrapers.length > 0) {
try { try {
logger.log('[LocalScraperService] Enabled scrapers:', enabledScrapers.map(s => s.name).join(', ')); logger.log('[LocalScraperService] Enabled scrapers:', enabledScrapers.map(s => s.name).join(', '));
} catch {} } catch { }
} }
if (enabledScrapers.length === 0) { if (enabledScrapers.length === 0) {
@ -983,7 +1140,7 @@ class LocalScraperService {
promise.finally(() => { promise.finally(() => {
const current = this.inFlightByKey.get(flightKey); const current = this.inFlightByKey.get(flightKey);
if (current === promise) this.inFlightByKey.delete(flightKey); if (current === promise) this.inFlightByKey.delete(flightKey);
}).catch(() => {}); }).catch(() => { });
} }
const results = await promise; const results = await promise;
@ -1013,16 +1170,16 @@ class LocalScraperService {
const settingsData = await mmkvStorage.getItem('app_settings'); const settingsData = await mmkvStorage.getItem('app_settings');
const settings = settingsData ? JSON.parse(settingsData) : {}; const settings = settingsData ? JSON.parse(settingsData) : {};
const urlValidationEnabled = settings.enableScraperUrlValidation ?? true; const urlValidationEnabled = settings.enableScraperUrlValidation ?? true;
// Load per-scraper settings for this run // Load per-scraper settings for this run
const allScraperSettingsRaw = await mmkvStorage.getItem(this.SCRAPER_SETTINGS_KEY); const allScraperSettingsRaw = await mmkvStorage.getItem(this.SCRAPER_SETTINGS_KEY);
const allScraperSettings = allScraperSettingsRaw ? JSON.parse(allScraperSettingsRaw) : {}; const allScraperSettings = allScraperSettingsRaw ? JSON.parse(allScraperSettingsRaw) : {};
const perScraperSettings = (params && params.scraperId && allScraperSettings[params.scraperId]) ? allScraperSettings[params.scraperId] : (params?.settings || {}); const perScraperSettings = (params && params.scraperId && allScraperSettings[params.scraperId]) ? allScraperSettings[params.scraperId] : (params?.settings || {});
// Create a limited global context // Create a limited global context
const moduleExports = {}; const moduleExports = {};
const moduleObj = { exports: moduleExports }; const moduleObj = { exports: moduleExports };
// Try to load cheerio-without-node-native // Try to load cheerio-without-node-native
let cheerio = null; let cheerio = null;
try { try {
@ -1034,14 +1191,14 @@ class LocalScraperService {
// Cheerio not available, scrapers will fall back to regex // Cheerio not available, scrapers will fall back to regex
} }
} }
// MovieBox constants - read from Expo public envs so they bundle in builds // MovieBox constants - read from Expo public envs so they bundle in builds
const MOVIEBOX_PRIMARY_KEY = process.env.EXPO_PUBLIC_MOVIEBOX_PRIMARY_KEY; const MOVIEBOX_PRIMARY_KEY = process.env.EXPO_PUBLIC_MOVIEBOX_PRIMARY_KEY;
const MOVIEBOX_TMDB_API_KEY = process.env.EXPO_PUBLIC_MOVIEBOX_TMDB_API_KEY || '439c478a771f35c05022f9feabcca01c'; const MOVIEBOX_TMDB_API_KEY = process.env.EXPO_PUBLIC_MOVIEBOX_TMDB_API_KEY || '439c478a771f35c05022f9feabcca01c';
if (!MOVIEBOX_PRIMARY_KEY) { if (!MOVIEBOX_PRIMARY_KEY) {
throw new Error('Missing EXPO_PUBLIC_MOVIEBOX_PRIMARY_KEY'); throw new Error('Missing EXPO_PUBLIC_MOVIEBOX_PRIMARY_KEY');
} }
const sandbox = { const sandbox = {
console: { console: {
log: (...args: any[]) => logger.log('[Scraper]', ...args), log: (...args: any[]) => logger.log('[Scraper]', ...args),
@ -1076,7 +1233,7 @@ class LocalScraperService {
// Add fetch for HTTP requests (using native fetch for MovieBox, axios for others) // Add fetch for HTTP requests (using native fetch for MovieBox, axios for others)
fetch: async (url: string, options: any = {}) => { fetch: async (url: string, options: any = {}) => {
const isMovieBoxRequest = url.includes('api.inmoviebox.com') || url.includes('themoviedb.org'); const isMovieBoxRequest = url.includes('api.inmoviebox.com') || url.includes('themoviedb.org');
if (isMovieBoxRequest) { if (isMovieBoxRequest) {
// Always use native fetch for MovieBox requests // Always use native fetch for MovieBox requests
try { try {
@ -1084,7 +1241,7 @@ class LocalScraperService {
method: options.method || 'GET', method: options.method || 'GET',
hasBody: !!options.body hasBody: !!options.body
}); });
const nativeResponse = await fetch(url, { const nativeResponse = await fetch(url, {
method: options.method || 'GET', method: options.method || 'GET',
headers: { headers: {
@ -1094,13 +1251,13 @@ class LocalScraperService {
}, },
body: options.body body: options.body
}); });
const responseData = await nativeResponse.text(); const responseData = await nativeResponse.text();
logger.log(`[Sandbox] Native fetch successful for MovieBox:`, { logger.log(`[Sandbox] Native fetch successful for MovieBox:`, {
status: nativeResponse.status, status: nativeResponse.status,
ok: nativeResponse.ok ok: nativeResponse.ok
}); });
return { return {
ok: nativeResponse.ok, ok: nativeResponse.ok,
status: nativeResponse.status, status: nativeResponse.status,
@ -1134,7 +1291,7 @@ class LocalScraperService {
timeout: 120000, // Increased to 2 minutes for complex scrapers timeout: 120000, // Increased to 2 minutes for complex scrapers
validateStatus: () => true // Don't throw on HTTP error status codes validateStatus: () => true // Don't throw on HTTP error status codes
}; };
try { try {
logger.log(`[Sandbox] Using axios for request: ${url}`, { logger.log(`[Sandbox] Using axios for request: ${url}`, {
method: axiosConfig.method, method: axiosConfig.method,
@ -1142,7 +1299,7 @@ class LocalScraperService {
hasBody: !!axiosConfig.data hasBody: !!axiosConfig.data
}); });
const response = await axios(axiosConfig); const response = await axios(axiosConfig);
return { return {
ok: response.status >= 200 && response.status < 300, ok: response.status >= 200 && response.status < 300,
status: response.status, status: response.status,
@ -1181,7 +1338,7 @@ class LocalScraperService {
SCRAPER_SETTINGS: perScraperSettings, SCRAPER_SETTINGS: perScraperSettings,
SCRAPER_ID: params?.scraperId SCRAPER_ID: params?.scraperId
}; };
// Execute the scraper code with 1 minute timeout // Execute the scraper code with 1 minute timeout
const SCRAPER_EXECUTION_TIMEOUT_MS = 60000; // 1 minute const SCRAPER_EXECUTION_TIMEOUT_MS = 60000; // 1 minute
@ -1237,7 +1394,7 @@ class LocalScraperService {
setTimeout(() => reject(new Error(`Scraper execution timed out after ${SCRAPER_EXECUTION_TIMEOUT_MS}ms`)), SCRAPER_EXECUTION_TIMEOUT_MS) setTimeout(() => reject(new Error(`Scraper execution timed out after ${SCRAPER_EXECUTION_TIMEOUT_MS}ms`)), SCRAPER_EXECUTION_TIMEOUT_MS)
) )
]); ]);
} catch (error) { } catch (error) {
logger.error('[LocalScraperService] Sandbox execution failed:', error); logger.error('[LocalScraperService] Sandbox execution failed:', error);
throw error; throw error;
@ -1250,22 +1407,22 @@ class LocalScraperService {
logger.warn('[LocalScraperService] Scraper returned non-array result'); logger.warn('[LocalScraperService] Scraper returned non-array result');
return []; return [];
} }
return results.map((result, index) => { return results.map((result, index) => {
// Build title with quality information for UI compatibility // Build title with quality information for UI compatibility
let title = result.title || result.name || `${scraper.name} Stream ${index + 1}`; let title = result.title || result.name || `${scraper.name} Stream ${index + 1}`;
// Add quality to title if available and not already present // Add quality to title if available and not already present
if (result.quality && !title.includes(result.quality)) { if (result.quality && !title.includes(result.quality)) {
title = `${title} ${result.quality}`; title = `${title} ${result.quality}`;
} }
// Build name with quality information // Build name with quality information
let streamName = result.name || `${scraper.name}`; let streamName = result.name || `${scraper.name}`;
if (result.quality && !streamName.includes(result.quality)) { if (result.quality && !streamName.includes(result.quality)) {
streamName = `${streamName} - ${result.quality}`; streamName = `${streamName} - ${result.quality}`;
} }
const stream: Stream = { const stream: Stream = {
// Include quality in name field for proper display // Include quality in name field for proper display
name: streamName, name: streamName,
@ -1280,22 +1437,22 @@ class LocalScraperService {
bingeGroup: `local-scraper-${scraper.id}` bingeGroup: `local-scraper-${scraper.id}`
} }
}; };
// Add additional properties if available // Add additional properties if available
if (result.infoHash) { if (result.infoHash) {
stream.infoHash = result.infoHash; stream.infoHash = result.infoHash;
} }
// Preserve any additional fields from the scraper result // Preserve any additional fields from the scraper result
if (result.quality && !stream.quality) { if (result.quality && !stream.quality) {
stream.quality = result.quality; stream.quality = result.quality;
} }
// Pass headers from scraper result if available // Pass headers from scraper result if available
if (result.headers) { if (result.headers) {
stream.headers = result.headers; stream.headers = result.headers;
} }
return stream; return stream;
}).filter(stream => stream.url); // Filter out streams without URLs }).filter(stream => stream.url); // Filter out streams without URLs
} }
@ -1303,13 +1460,13 @@ class LocalScraperService {
// Parse size string to bytes // Parse size string to bytes
private parseSize(sizeStr: string): number { private parseSize(sizeStr: string): number {
if (!sizeStr) return 0; if (!sizeStr) return 0;
const match = sizeStr.match(/([0-9.]+)\s*(GB|MB|KB|TB)/i); const match = sizeStr.match(/([0-9.]+)\s*(GB|MB|KB|TB)/i);
if (!match) return 0; if (!match) return 0;
const value = parseFloat(match[1]); const value = parseFloat(match[1]);
const unit = match[2].toUpperCase(); const unit = match[2].toUpperCase();
switch (unit) { switch (unit) {
case 'TB': return value * 1024 * 1024 * 1024 * 1024; case 'TB': return value * 1024 * 1024 * 1024 * 1024;
case 'GB': return value * 1024 * 1024 * 1024; case 'GB': return value * 1024 * 1024 * 1024;
@ -1323,22 +1480,22 @@ class LocalScraperService {
async clearScrapers(): Promise<void> { async clearScrapers(): Promise<void> {
this.installedScrapers.clear(); this.installedScrapers.clear();
this.scraperCode.clear(); this.scraperCode.clear();
// Clear from storage // Clear from storage
await mmkvStorage.removeItem(this.STORAGE_KEY); await mmkvStorage.removeItem(this.STORAGE_KEY);
// Clear cached code // Clear cached code
const keys = await mmkvStorage.getAllKeys(); const keys = await mmkvStorage.getAllKeys();
const scraperCodeKeys = keys.filter(key => key.startsWith('scraper-code-')); const scraperCodeKeys = keys.filter(key => key.startsWith('scraper-code-'));
await mmkvStorage.multiRemove(scraperCodeKeys); await mmkvStorage.multiRemove(scraperCodeKeys);
logger.log('[LocalScraperService] All scrapers cleared'); logger.log('[LocalScraperService] All scrapers cleared');
} }
// Check if local scrapers are available // Check if local scrapers are available
async hasScrapers(): Promise<boolean> { async hasScrapers(): Promise<boolean> {
await this.ensureInitialized(); await this.ensureInitialized();
// Get user settings to check if local scrapers are enabled // Get user settings to check if local scrapers are enabled
const userSettings = await this.getUserScraperSettings(); const userSettings = await this.getUserScraperSettings();
logger.log('[LocalScraperService.hasScrapers] enableLocalScrapers:', userSettings.enableLocalScrapers); logger.log('[LocalScraperService.hasScrapers] enableLocalScrapers:', userSettings.enableLocalScrapers);
@ -1346,13 +1503,13 @@ class LocalScraperService {
logger.log('[LocalScraperService.hasScrapers] Returning false: local scrapers disabled'); logger.log('[LocalScraperService.hasScrapers] Returning false: local scrapers disabled');
return false; return false;
} }
// If no repository is configured, return false // If no repository is configured, return false
if (!this.repositoryUrl) { if (!this.repositoryUrl) {
logger.log('[LocalScraperService.hasScrapers] Returning false: no repository URL configured'); logger.log('[LocalScraperService.hasScrapers] Returning false: no repository URL configured');
return false; return false;
} }
// If no scrapers are installed, try to refresh repository // If no scrapers are installed, try to refresh repository
if (this.installedScrapers.size === 0) { if (this.installedScrapers.size === 0) {
logger.log('[LocalScraperService.hasScrapers] No scrapers installed, attempting to refresh repository'); logger.log('[LocalScraperService.hasScrapers] No scrapers installed, attempting to refresh repository');
@ -1363,16 +1520,16 @@ class LocalScraperService {
return false; return false;
} }
} }
logger.log('[LocalScraperService.hasScrapers] installedScrapers.size:', this.installedScrapers.size); logger.log('[LocalScraperService.hasScrapers] installedScrapers.size:', this.installedScrapers.size);
logger.log('[LocalScraperService.hasScrapers] enabledScrapers set size:', userSettings.enabledScrapers?.size); logger.log('[LocalScraperService.hasScrapers] enabledScrapers set size:', userSettings.enabledScrapers?.size);
// Check if there are any enabled scrapers based on user settings // Check if there are any enabled scrapers based on user settings
if (userSettings.enabledScrapers && userSettings.enabledScrapers.size > 0) { if (userSettings.enabledScrapers && userSettings.enabledScrapers.size > 0) {
logger.log('[LocalScraperService.hasScrapers] Returning true: enabledScrapers set has items'); logger.log('[LocalScraperService.hasScrapers] Returning true: enabledScrapers set has items');
return true; return true;
} }
// Fallback: check if any scrapers are enabled in the internal state // Fallback: check if any scrapers are enabled in the internal state
const hasEnabledScrapers = Array.from(this.installedScrapers.values()).some(scraper => scraper.enabled); const hasEnabledScrapers = Array.from(this.installedScrapers.values()).some(scraper => scraper.enabled);
logger.log('[LocalScraperService.hasScrapers] Fallback check - hasEnabledScrapers:', hasEnabledScrapers); logger.log('[LocalScraperService.hasScrapers] Fallback check - hasEnabledScrapers:', hasEnabledScrapers);