From e0182a700f310f5dfd6436b1e8555cb620b7bd90 Mon Sep 17 00:00:00 2001 From: kingbri Date: Tue, 23 Aug 2022 11:55:41 -0400 Subject: [PATCH] Sources: Add website API support Adds support for website APIs both complex and simple. This commit only supports GET requests to APIs. POST request support can be added on request. Client IDs and secrets are also supported. They can be added via source settings or automatically set by a website endpoint. Also fetch sources for scraping using the backgroundContext and remove some functions from using the main actor. Signed-off-by: kingbri --- Ferrite.xcodeproj/project.pbxproj | 25 + .../Classes/Source+CoreDataProperties.swift | 2 + ...ourceComplexQuery+CoreDataProperties.swift | 4 +- .../SourceHtmlParser+CoreDataProperties.swift | 32 +- .../SourceJsonParser+CoreDataClass.swift | 13 + .../SourceJsonParser+CoreDataProperties.swift | 28 + .../SourceRssParser+CoreDataProperties.swift | 34 +- .../SourceSeedLeech+CoreDataProperties.swift | 2 +- .../FerriteDB.xcdatamodel/contents | 47 +- Ferrite/Models/SourceModels.swift | 38 +- Ferrite/ViewModels/DebridManager.swift | 11 +- Ferrite/ViewModels/NavigationViewModel.swift | 13 +- Ferrite/ViewModels/ScrapingViewModel.swift | 507 ++++++++++++++---- Ferrite/ViewModels/SourceManager.swift | 125 ++++- Ferrite/Views/ContentView.swift | 9 +- Ferrite/Views/MagnetChoiceView.swift | 5 +- Ferrite/Views/SearchResultRDView.swift | 4 +- Ferrite/Views/SearchResultsView.swift | 2 +- .../SourceViews/SourceSettingsView.swift | 29 +- 19 files changed, 737 insertions(+), 193 deletions(-) create mode 100644 Ferrite/DataManagement/Classes/SourceJsonParser+CoreDataClass.swift create mode 100644 Ferrite/DataManagement/Classes/SourceJsonParser+CoreDataProperties.swift diff --git a/Ferrite.xcodeproj/project.pbxproj b/Ferrite.xcodeproj/project.pbxproj index 4cc8237..19c9841 100644 --- a/Ferrite.xcodeproj/project.pbxproj +++ b/Ferrite.xcodeproj/project.pbxproj @@ -9,6 +9,8 @@ /* Begin PBXBuildFile section */ 0C0D50E5288DFE7F0035ECC8 /* SourceModels.swift in Sources */ = {isa = PBXBuildFile; fileRef = 0C0D50E4288DFE7F0035ECC8 /* SourceModels.swift */; }; 0C0D50E7288DFF850035ECC8 /* SourcesView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 0C0D50E6288DFF850035ECC8 /* SourcesView.swift */; }; + 0C31133C28B1ABFA004DCB0D /* SourceJsonParser+CoreDataClass.swift in Sources */ = {isa = PBXBuildFile; fileRef = 0C31133A28B1ABFA004DCB0D /* SourceJsonParser+CoreDataClass.swift */; }; + 0C31133D28B1ABFA004DCB0D /* SourceJsonParser+CoreDataProperties.swift in Sources */ = {isa = PBXBuildFile; fileRef = 0C31133B28B1ABFA004DCB0D /* SourceJsonParser+CoreDataProperties.swift */; }; 0C32FB532890D19D002BD219 /* AboutView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 0C32FB522890D19D002BD219 /* AboutView.swift */; }; 0C32FB552890D1BF002BD219 /* UIApplication.swift in Sources */ = {isa = PBXBuildFile; fileRef = 0C32FB542890D1BF002BD219 /* UIApplication.swift */; }; 0C32FB572890D1F2002BD219 /* ListRowViews.swift in Sources */ = {isa = PBXBuildFile; fileRef = 0C32FB562890D1F2002BD219 /* ListRowViews.swift */; }; @@ -21,6 +23,7 @@ 0C64A4B7288903880079976D /* KeychainSwift in Frameworks */ = {isa = PBXBuildFile; productRef = 0C64A4B6288903880079976D /* KeychainSwift */; }; 0C733287289C4C820058D1FE /* SourceSettingsView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 0C733286289C4C820058D1FE /* SourceSettingsView.swift */; }; 0C7376F028A97D1400D60918 /* SwiftUIX in Frameworks */ = {isa = PBXBuildFile; productRef = 0C7376EF28A97D1400D60918 /* SwiftUIX */; }; + 0C7506D728B1AC9A008BEE38 /* SwiftyJSON in Frameworks */ = {isa = PBXBuildFile; productRef = 0C7506D628B1AC9A008BEE38 /* SwiftyJSON */; }; 0C750744289B003E004B3906 /* SourceRssParser+CoreDataClass.swift in Sources */ = {isa = PBXBuildFile; fileRef = 0C750742289B003E004B3906 /* SourceRssParser+CoreDataClass.swift */; }; 0C750745289B003E004B3906 /* SourceRssParser+CoreDataProperties.swift in Sources */ = {isa = PBXBuildFile; fileRef = 0C750743289B003E004B3906 /* SourceRssParser+CoreDataProperties.swift */; }; 0C794B67289DACB600DD1CC8 /* SourceUpdateButtonView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 0C794B66289DACB600DD1CC8 /* SourceUpdateButtonView.swift */; }; @@ -72,6 +75,8 @@ /* Begin PBXFileReference section */ 0C0D50E4288DFE7F0035ECC8 /* SourceModels.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SourceModels.swift; sourceTree = ""; }; 0C0D50E6288DFF850035ECC8 /* SourcesView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SourcesView.swift; sourceTree = ""; }; + 0C31133A28B1ABFA004DCB0D /* SourceJsonParser+CoreDataClass.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "SourceJsonParser+CoreDataClass.swift"; sourceTree = ""; }; + 0C31133B28B1ABFA004DCB0D /* SourceJsonParser+CoreDataProperties.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "SourceJsonParser+CoreDataProperties.swift"; sourceTree = ""; }; 0C32FB522890D19D002BD219 /* AboutView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AboutView.swift; sourceTree = ""; }; 0C32FB542890D1BF002BD219 /* UIApplication.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = UIApplication.swift; sourceTree = ""; }; 0C32FB562890D1F2002BD219 /* ListRowViews.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ListRowViews.swift; sourceTree = ""; }; @@ -134,6 +139,7 @@ isa = PBXFrameworksBuildPhase; buildActionMask = 2147483647; files = ( + 0C7506D728B1AC9A008BEE38 /* SwiftyJSON in Frameworks */, 0C64A4B4288903680079976D /* Base32 in Frameworks */, 0C4CFC462897030D00AD9FAD /* Regex in Frameworks */, 0C7376F028A97D1400D60918 /* SwiftUIX in Frameworks */, @@ -148,6 +154,8 @@ 0C0D50DE288DF72D0035ECC8 /* Classes */ = { isa = PBXGroup; children = ( + 0C31133A28B1ABFA004DCB0D /* SourceJsonParser+CoreDataClass.swift */, + 0C31133B28B1ABFA004DCB0D /* SourceJsonParser+CoreDataProperties.swift */, 0C750742289B003E004B3906 /* SourceRssParser+CoreDataClass.swift */, 0C750743289B003E004B3906 /* SourceRssParser+CoreDataProperties.swift */, 0C79DC052899AF3C003F1C5A /* SourceSeedLeech+CoreDataClass.swift */, @@ -342,6 +350,7 @@ 0C64A4B6288903880079976D /* KeychainSwift */, 0C4CFC452897030D00AD9FAD /* Regex */, 0C7376EF28A97D1400D60918 /* SwiftUIX */, + 0C7506D628B1AC9A008BEE38 /* SwiftyJSON */, ); productName = Torrenter; productReference = 0CAF1C68286F5C0E00296F86 /* Ferrite.app */; @@ -377,6 +386,7 @@ 0C64A4B5288903880079976D /* XCRemoteSwiftPackageReference "keychain-swift" */, 0C4CFC442897030D00AD9FAD /* XCRemoteSwiftPackageReference "Regex" */, 0C7376EE28A97D1400D60918 /* XCRemoteSwiftPackageReference "SwiftUIX" */, + 0C7506D528B1AC9A008BEE38 /* XCRemoteSwiftPackageReference "SwiftyJSON" */, ); productRefGroup = 0CAF1C69286F5C0E00296F86 /* Products */; projectDirPath = ""; @@ -412,6 +422,7 @@ 0CA148DB288903F000DE2211 /* NavView.swift in Sources */, 0C750745289B003E004B3906 /* SourceRssParser+CoreDataProperties.swift in Sources */, 0CBC7705288DE7F40054BE44 /* PersistenceController.swift in Sources */, + 0C31133D28B1ABFA004DCB0D /* SourceJsonParser+CoreDataProperties.swift in Sources */, 0CA0545B288EEA4E00850554 /* SourceListEditorView.swift in Sources */, 0C84F4872895BFED0074B7C9 /* SourceList+CoreDataProperties.swift in Sources */, 0C794B6B289DACF100DD1CC8 /* SourceCatalogView.swift in Sources */, @@ -444,6 +455,7 @@ 0C84F4772895BE680074B7C9 /* FerriteDB.xcdatamodeld in Sources */, 0C733287289C4C820058D1FE /* SourceSettingsView.swift in Sources */, 0CA05457288EE58200850554 /* SettingsSourceListView.swift in Sources */, + 0C31133C28B1ABFA004DCB0D /* SourceJsonParser+CoreDataClass.swift in Sources */, 0CA148DE288903F000DE2211 /* RealDebridModels.swift in Sources */, 0CBC76FF288DAAD00054BE44 /* NavigationViewModel.swift in Sources */, 0C4CFC4D28970C8B00AD9FAD /* SourceComplexQuery+CoreDataClass.swift in Sources */, @@ -699,6 +711,14 @@ kind = branch; }; }; + 0C7506D528B1AC9A008BEE38 /* XCRemoteSwiftPackageReference "SwiftyJSON" */ = { + isa = XCRemoteSwiftPackageReference; + repositoryURL = "https://github.com/SwiftyJSON/SwiftyJSON"; + requirement = { + branch = master; + kind = branch; + }; + }; 0CAF1C79286F5C8600296F86 /* XCRemoteSwiftPackageReference "SwiftSoup" */ = { isa = XCRemoteSwiftPackageReference; repositoryURL = "https://github.com/scinfu/SwiftSoup.git"; @@ -730,6 +750,11 @@ package = 0C7376EE28A97D1400D60918 /* XCRemoteSwiftPackageReference "SwiftUIX" */; productName = SwiftUIX; }; + 0C7506D628B1AC9A008BEE38 /* SwiftyJSON */ = { + isa = XCSwiftPackageProductDependency; + package = 0C7506D528B1AC9A008BEE38 /* XCRemoteSwiftPackageReference "SwiftyJSON" */; + productName = SwiftyJSON; + }; 0CAF1C7A286F5C8600296F86 /* SwiftSoup */ = { isa = XCSwiftPackageProductDependency; package = 0CAF1C79286F5C8600296F86 /* XCRemoteSwiftPackageReference "SwiftSoup" */; diff --git a/Ferrite/DataManagement/Classes/Source+CoreDataProperties.swift b/Ferrite/DataManagement/Classes/Source+CoreDataProperties.swift index dcaf8fc..4a1a0cf 100644 --- a/Ferrite/DataManagement/Classes/Source+CoreDataProperties.swift +++ b/Ferrite/DataManagement/Classes/Source+CoreDataProperties.swift @@ -25,7 +25,9 @@ public extension Source { @NSManaged var version: Int16 @NSManaged var htmlParser: SourceHtmlParser? @NSManaged var rssParser: SourceRssParser? + @NSManaged var jsonParser: SourceJsonParser? @NSManaged var api: SourceApi? + @NSManaged var trackers: [String]? } extension Source: Identifiable {} diff --git a/Ferrite/DataManagement/Classes/SourceComplexQuery+CoreDataProperties.swift b/Ferrite/DataManagement/Classes/SourceComplexQuery+CoreDataProperties.swift index 56b25f8..0f4d16e 100644 --- a/Ferrite/DataManagement/Classes/SourceComplexQuery+CoreDataProperties.swift +++ b/Ferrite/DataManagement/Classes/SourceComplexQuery+CoreDataProperties.swift @@ -2,7 +2,7 @@ // SourceComplexQuery+CoreDataProperties.swift // Ferrite // -// Created by Brian Dashore on 7/31/22. +// Created by Brian Dashore on 8/22/22. // // @@ -15,7 +15,7 @@ public extension SourceComplexQuery { } @NSManaged var attribute: String - @NSManaged var lookupAttribute: String? + @NSManaged var discriminator: String? @NSManaged var query: String @NSManaged var regex: String? } diff --git a/Ferrite/DataManagement/Classes/SourceHtmlParser+CoreDataProperties.swift b/Ferrite/DataManagement/Classes/SourceHtmlParser+CoreDataProperties.swift index 4a1f025..1cc03aa 100644 --- a/Ferrite/DataManagement/Classes/SourceHtmlParser+CoreDataProperties.swift +++ b/Ferrite/DataManagement/Classes/SourceHtmlParser+CoreDataProperties.swift @@ -6,28 +6,22 @@ // // -import Foundation import CoreData +import Foundation - -extension SourceHtmlParser { - - @nonobjc public class func fetchRequest() -> NSFetchRequest { - return NSFetchRequest(entityName: "SourceHtmlParser") +public extension SourceHtmlParser { + @nonobjc class func fetchRequest() -> NSFetchRequest { + NSFetchRequest(entityName: "SourceHtmlParser") } - @NSManaged public var rows: String - @NSManaged public var searchUrl: String - @NSManaged public var trackers: [String]? - @NSManaged public var magnetHash: SourceMagnetHash? - @NSManaged public var magnetLink: SourceMagnetLink? - @NSManaged public var parentSource: Source? - @NSManaged public var seedLeech: SourceSeedLeech? - @NSManaged public var size: SourceSize? - @NSManaged public var title: SourceTitle? - + @NSManaged var rows: String + @NSManaged var searchUrl: String + @NSManaged var magnetHash: SourceMagnetHash? + @NSManaged var magnetLink: SourceMagnetLink? + @NSManaged var parentSource: Source? + @NSManaged var seedLeech: SourceSeedLeech? + @NSManaged var size: SourceSize? + @NSManaged var title: SourceTitle? } -extension SourceHtmlParser : Identifiable { - -} +extension SourceHtmlParser: Identifiable {} diff --git a/Ferrite/DataManagement/Classes/SourceJsonParser+CoreDataClass.swift b/Ferrite/DataManagement/Classes/SourceJsonParser+CoreDataClass.swift new file mode 100644 index 0000000..055f0c6 --- /dev/null +++ b/Ferrite/DataManagement/Classes/SourceJsonParser+CoreDataClass.swift @@ -0,0 +1,13 @@ +// +// SourceJsonParser+CoreDataClass.swift +// Ferrite +// +// Created by Brian Dashore on 8/20/22. +// +// + +import CoreData +import Foundation + +@objc(SourceJsonParser) +public class SourceJsonParser: NSManagedObject {} diff --git a/Ferrite/DataManagement/Classes/SourceJsonParser+CoreDataProperties.swift b/Ferrite/DataManagement/Classes/SourceJsonParser+CoreDataProperties.swift new file mode 100644 index 0000000..84de4d3 --- /dev/null +++ b/Ferrite/DataManagement/Classes/SourceJsonParser+CoreDataProperties.swift @@ -0,0 +1,28 @@ +// +// SourceJsonParser+CoreDataProperties.swift +// Ferrite +// +// Created by Brian Dashore on 8/21/22. +// +// + +import CoreData +import Foundation + +public extension SourceJsonParser { + @nonobjc class func fetchRequest() -> NSFetchRequest { + NSFetchRequest(entityName: "SourceJsonParser") + } + + @NSManaged var results: String? + @NSManaged var subResults: String? + @NSManaged var searchUrl: String + @NSManaged var magnetHash: SourceMagnetHash? + @NSManaged var magnetLink: SourceMagnetLink? + @NSManaged var parentSource: Source? + @NSManaged var seedLeech: SourceSeedLeech? + @NSManaged var size: SourceSize? + @NSManaged var title: SourceTitle? +} + +extension SourceJsonParser: Identifiable {} diff --git a/Ferrite/DataManagement/Classes/SourceRssParser+CoreDataProperties.swift b/Ferrite/DataManagement/Classes/SourceRssParser+CoreDataProperties.swift index 1ef4e89..eef0e62 100644 --- a/Ferrite/DataManagement/Classes/SourceRssParser+CoreDataProperties.swift +++ b/Ferrite/DataManagement/Classes/SourceRssParser+CoreDataProperties.swift @@ -6,29 +6,23 @@ // // -import Foundation import CoreData +import Foundation - -extension SourceRssParser { - - @nonobjc public class func fetchRequest() -> NSFetchRequest { - return NSFetchRequest(entityName: "SourceRssParser") +public extension SourceRssParser { + @nonobjc class func fetchRequest() -> NSFetchRequest { + NSFetchRequest(entityName: "SourceRssParser") } - @NSManaged public var items: String - @NSManaged public var rssUrl: String? - @NSManaged public var searchUrl: String - @NSManaged public var trackers: [String]? - @NSManaged public var magnetHash: SourceMagnetHash? - @NSManaged public var magnetLink: SourceMagnetLink? - @NSManaged public var parentSource: Source? - @NSManaged public var seedLeech: SourceSeedLeech? - @NSManaged public var size: SourceSize? - @NSManaged public var title: SourceTitle? - + @NSManaged var items: String + @NSManaged var rssUrl: String? + @NSManaged var searchUrl: String + @NSManaged var magnetHash: SourceMagnetHash? + @NSManaged var magnetLink: SourceMagnetLink? + @NSManaged var parentSource: Source? + @NSManaged var seedLeech: SourceSeedLeech? + @NSManaged var size: SourceSize? + @NSManaged var title: SourceTitle? } -extension SourceRssParser : Identifiable { - -} +extension SourceRssParser: Identifiable {} diff --git a/Ferrite/DataManagement/Classes/SourceSeedLeech+CoreDataProperties.swift b/Ferrite/DataManagement/Classes/SourceSeedLeech+CoreDataProperties.swift index b2362a0..acd0173 100644 --- a/Ferrite/DataManagement/Classes/SourceSeedLeech+CoreDataProperties.swift +++ b/Ferrite/DataManagement/Classes/SourceSeedLeech+CoreDataProperties.swift @@ -20,7 +20,7 @@ public extension SourceSeedLeech { @NSManaged var seederRegex: String? @NSManaged var seeders: String? @NSManaged var attribute: String - @NSManaged var lookupAttribute: String? + @NSManaged var discriminator: String? @NSManaged var parentParser: SourceHtmlParser? } diff --git a/Ferrite/DataManagement/FerriteDB.xcdatamodeld/FerriteDB.xcdatamodel/contents b/Ferrite/DataManagement/FerriteDB.xcdatamodeld/FerriteDB.xcdatamodel/contents index adf97bb..c2b3812 100644 --- a/Ferrite/DataManagement/FerriteDB.xcdatamodeld/FerriteDB.xcdatamodel/contents +++ b/Ferrite/DataManagement/FerriteDB.xcdatamodeld/FerriteDB.xcdatamodel/contents @@ -9,34 +9,61 @@ + + - - - + + + + + + + + + + + + + + + + + + - + - - + + + + + + + + + + + + @@ -45,18 +72,19 @@ + + - @@ -67,20 +95,23 @@ + - + + + \ No newline at end of file diff --git a/Ferrite/Models/SourceModels.swift b/Ferrite/Models/SourceModels.swift index 1d153e9..daa30ca 100644 --- a/Ferrite/Models/SourceModels.swift +++ b/Ferrite/Models/SourceModels.swift @@ -7,6 +7,11 @@ import Foundation +public enum ApiCredentialResponseType: String, Codable, Hashable { + case json + case text +} + public struct SourceListJson: Codable { let name: String let author: String @@ -20,7 +25,9 @@ public struct SourceJson: Codable, Hashable { var dynamicBaseUrl: Bool? var author: String? var listId: UUID? + let trackers: [String]? let api: SourceApiJson? + let jsonParser: SourceJsonParserJson? let rssParser: SourceRssParserJson? let htmlParser: SourceHtmlParserJson? } @@ -33,9 +40,29 @@ public enum SourcePreferredParser: Int16, CaseIterable { } public struct SourceApiJson: Codable, Hashable { - let clientId: String? - var dynamicClientId: Bool? - let usesSecret: Bool + let apiUrl: String? + let clientId: SourceApiCredentialJson? + let clientSecret: SourceApiCredentialJson? +} + +public struct SourceApiCredentialJson: Codable, Hashable { + let query: String? + let value: String? + let dynamic: Bool? + let url: String? + let responseType: ApiCredentialResponseType? + let expiryLength: Double? +} + +public struct SourceJsonParserJson: Codable, Hashable { + let searchUrl: String + let results: String? + let subResults: String? + let magnetHash: SouceComplexQueryJson? + let magnetLink: SouceComplexQueryJson? + let title: SouceComplexQueryJson? + let size: SouceComplexQueryJson? + let sl: SourceSLJson? } public struct SourceRssParserJson: Codable, Hashable { @@ -47,7 +74,6 @@ public struct SourceRssParserJson: Codable, Hashable { let title: SouceComplexQueryJson? let size: SouceComplexQueryJson? let sl: SourceSLJson? - let trackers: [String]? } public struct SourceHtmlParserJson: Codable, Hashable { @@ -61,7 +87,7 @@ public struct SourceHtmlParserJson: Codable, Hashable { public struct SouceComplexQueryJson: Codable, Hashable { let query: String - let lookupAttribute: String? + let discriminator: String? let attribute: String? let regex: String? } @@ -78,7 +104,7 @@ public struct SourceSLJson: Codable, Hashable { let leechers: String? let combined: String? let attribute: String? - let lookupAttribute: String? + let discriminator: String? let seederRegex: String? let leecherRegex: String? } diff --git a/Ferrite/ViewModels/DebridManager.swift b/Ferrite/ViewModels/DebridManager.swift index 5130d5c..5e098cc 100644 --- a/Ferrite/ViewModels/DebridManager.swift +++ b/Ferrite/ViewModels/DebridManager.swift @@ -106,8 +106,17 @@ public class DebridManager: ObservableObject { } public func fetchRdDownload(searchResult: SearchResult, iaFile: RealDebridIAFile? = nil) async { + guard let magnetLink = searchResult.magnetLink else { + Task { @MainActor in + toastModel?.toastDescription = "Could not run your action because the magnet link is invalid." + } + print("RD error: Invalid magnet link") + + return + } + do { - let realDebridId = try await realDebrid.addMagnet(magnetLink: searchResult.magnetLink) + let realDebridId = try await realDebrid.addMagnet(magnetLink: magnetLink) var fileIds: [Int] = [] diff --git a/Ferrite/ViewModels/NavigationViewModel.swift b/Ferrite/ViewModels/NavigationViewModel.swift index ab97121..de84fc2 100644 --- a/Ferrite/ViewModels/NavigationViewModel.swift +++ b/Ferrite/ViewModels/NavigationViewModel.swift @@ -86,17 +86,26 @@ class NavigationViewModel: ObservableObject { public func runMagnetAction(action: DefaultMagnetActionType?, searchResult: SearchResult) { let selectedAction = action ?? defaultMagnetAction + guard let magnetLink = searchResult.magnetLink else { + toastModel?.toastDescription = "Could not run your action because the magnet link is invalid." + print("Magnet action error: The magnet link is invalid.") + + return + } + switch selectedAction { case .none: currentChoiceSheet = .magnet case .webtor: - if let url = URL(string: "https://webtor.io/#/show?magnet=\(searchResult.magnetLink)") { + if let url = URL(string: "https://webtor.io/#/show?magnet=\(magnetLink)") { UIApplication.shared.open(url) } else { toastModel?.toastDescription = "Could not create a WebTor URL" } case .shareMagnet: - if let magnetUrl = URL(string: searchResult.magnetLink), currentChoiceSheet == nil { + if let magnetUrl = URL(string: magnetLink), + currentChoiceSheet == nil + { activityItems = [magnetUrl] showActivityView.toggle() } else { diff --git a/Ferrite/ViewModels/ScrapingViewModel.swift b/Ferrite/ViewModels/ScrapingViewModel.swift index e17c043..4beec42 100644 --- a/Ferrite/ViewModels/ScrapingViewModel.swift +++ b/Ferrite/ViewModels/ScrapingViewModel.swift @@ -9,12 +9,13 @@ import Base32 import Regex import SwiftSoup import SwiftUI +import SwiftyJSON public struct SearchResult: Hashable, Codable { - let title: String + let title: String? let source: String - let size: String - let magnetLink: String + let size: String? + let magnetLink: String? let magnetHash: String? let seeders: String? let leechers: String? @@ -42,7 +43,7 @@ class ScrapingViewModel: ObservableObject { toastModel?.toastDescription = "There are no sources to search!" } - print("Sources empty") + print("There are no sources to search!") return } @@ -53,9 +54,7 @@ class ScrapingViewModel: ObservableObject { currentSourceName = source.name guard let baseUrl = source.baseUrl else { - Task { @MainActor in - toastModel?.toastDescription = "The base URL could not be found for source \(source.name)" - } + toastModel?.toastDescription = "The base URL could not be found for source \(source.name)" print("The base URL could not be found for source \(source.name)") continue @@ -64,54 +63,82 @@ class ScrapingViewModel: ObservableObject { // Default to HTML scraping let preferredParser = SourcePreferredParser(rawValue: source.preferredParser) ?? .none + guard let encodedQuery = searchText.addingPercentEncoding(withAllowedCharacters: .urlPathAllowed) else { + toastModel?.toastDescription = "Could not process search query, invalid characters present." + print("Could not process search query, invalid characters present") + + continue + } + switch preferredParser { case .scraping: if let htmlParser = source.htmlParser { - guard let encodedQuery = searchText.addingPercentEncoding(withAllowedCharacters: .urlPathAllowed) else { - toastModel?.toastDescription = "Could not process search query, invalid characters present." - print("Could not process search query, invalid characters present") + let urlString = baseUrl + htmlParser.searchUrl + .replacingOccurrences(of: "{query}", with: encodedQuery) - continue + if let data = await fetchWebsiteData(urlString: urlString), + let html = String(data: data, encoding: .utf8) + { + let sourceResults = await scrapeHtml(source: source, baseUrl: baseUrl, html: html) + tempResults += sourceResults } - - let urlString = baseUrl + htmlParser.searchUrl.replacingOccurrences(of: "{query}", with: encodedQuery) - - guard let html = await fetchWebsiteData(urlString: urlString) else { - continue - } - - let sourceResults = await scrapeHtml(source: source, baseUrl: baseUrl, html: html) - tempResults += sourceResults } case .rss: if let rssParser = source.rssParser { - guard let encodedQuery = searchText.addingPercentEncoding(withAllowedCharacters: .urlPathAllowed) else { - toastModel?.toastDescription = "Could not process search query, invalid characters present." - print("Could not process search query, invalid characters present") - - continue - } - let replacedSearchUrl = rssParser.searchUrl - .replacingOccurrences(of: "{apiKey}", with: source.api?.clientSecret ?? "") + .replacingOccurrences(of: "{secret}", with: source.api?.clientSecret?.value ?? "") .replacingOccurrences(of: "{query}", with: encodedQuery) // If there is an RSS base URL, use that instead - var urlString: String - if let rssUrl = rssParser.rssUrl { - urlString = rssUrl + replacedSearchUrl - } else { - urlString = baseUrl + replacedSearchUrl - } + let urlString = (rssParser.rssUrl ?? baseUrl) + replacedSearchUrl - guard let rss = await fetchWebsiteData(urlString: urlString) else { - continue + if let data = await fetchWebsiteData(urlString: urlString), + let rss = String(data: data, encoding: .utf8) + { + let sourceResults = scrapeRss(source: source, rss: rss) + tempResults += sourceResults } - - let sourceResults = scrapeRss(source: source, rss: rss) - tempResults += sourceResults } - case .siteApi, .none: + case .siteApi: + if let jsonParser = source.jsonParser { + var replacedSearchUrl = jsonParser.searchUrl + .replacingOccurrences(of: "{query}", with: encodedQuery) + + // Handle anything API related including tokens, client IDs, and appending the API URL + // The source API key is for APIs that require extra credentials or use a different URL + if let sourceApi = source.api { + if let clientIdInfo = sourceApi.clientId { + if let newSearchUrl = await handleApiCredential(clientIdInfo, + replacement: "{clientId}", + searchUrl: replacedSearchUrl, + apiUrl: sourceApi.apiUrl, + baseUrl: baseUrl) + { + replacedSearchUrl = newSearchUrl + } + } + + // Works exactly the same as the client ID check + if let clientSecretInfo = sourceApi.clientSecret { + if let newSearchUrl = await handleApiCredential(clientSecretInfo, + replacement: "{secret}", + searchUrl: replacedSearchUrl, + apiUrl: sourceApi.apiUrl, + baseUrl: baseUrl) + { + replacedSearchUrl = newSearchUrl + } + } + } + + let urlString = (source.api?.apiUrl ?? baseUrl) + replacedSearchUrl + + if let data = await fetchWebsiteData(urlString: urlString) { + let sourceResults = scrapeJson(source: source, jsonData: data) + tempResults += sourceResults + } + } + case .none: continue } } @@ -125,11 +152,90 @@ class ScrapingViewModel: ObservableObject { searchResults = tempResults } - // Fetches the data for a URL - @MainActor - public func fetchWebsiteData(urlString: String) async -> String? { + public func handleApiCredential(_ credential: SourceApiCredential, + replacement: String, + searchUrl: String, + apiUrl: String?, + baseUrl: String) async -> String? + { + // Is the credential expired + var isExpired = false + if let timeStamp = credential.timeStamp?.timeIntervalSince1970, credential.expiryLength != 0 { + let now = Date().timeIntervalSince1970 + + isExpired = now > timeStamp + credential.expiryLength + } + + // Fetch a new credential if it's expired or doesn't exist yet + if let value = credential.value, !isExpired { + return searchUrl + .replacingOccurrences(of: replacement, with: value) + } else if + credential.value == nil || isExpired, + let credentialUrl = credential.urlString, + let newValue = await fetchApiCredential( + urlString: (apiUrl ?? baseUrl) + credentialUrl, + credential: credential + ) + { + let backgroundContext = PersistenceController.shared.backgroundContext + + credential.value = newValue + credential.timeStamp = Date() + + PersistenceController.shared.save(backgroundContext) + + return searchUrl + .replacingOccurrences(of: replacement, with: newValue) + } + + return nil + } + + public func fetchApiCredential(urlString: String, credential: SourceApiCredential) async -> String? { guard let url = URL(string: urlString) else { - toastModel?.toastDescription = "Source doesn't contain a valid URL, contact the source dev!" + Task { @MainActor in + toastModel?.toastDescription = "This token URL is invalid." + } + print("Token url \(urlString) is invalid!") + + return nil + } + + do { + let (data, _) = try await URLSession.shared.data(from: url) + + let responseType = ApiCredentialResponseType(rawValue: credential.responseType ?? "") ?? .json + + switch responseType { + case .json: + guard let credentialQuery = credential.query else { + return nil + } + + let json = try JSON(data: data) + + return json[credentialQuery.components(separatedBy: ".")].string + case .text: + return String(data: data, encoding: .utf8) + } + } catch { + Task { @MainActor in + toastModel?.toastDescription = "Error in fetching an API credential \(error)" + } + print("Error in fetching an API credential \(error)") + + return nil + } + } + + // Fetches the data for a URL + public func fetchWebsiteData(urlString: String) async -> Data? { + guard let url = URL(string: urlString) else { + Task { @MainActor in + toastModel?.toastDescription = "Source doesn't contain a valid URL, contact the source dev!" + } + print("Source doesn't contain a valid URL, contact the source dev!") return nil @@ -137,27 +243,182 @@ class ScrapingViewModel: ObservableObject { do { let (data, _) = try await URLSession.shared.data(from: url) - let html = String(data: data, encoding: .ascii) - return html + return data } catch { let error = error as NSError - switch error.code { - case -999: - toastModel?.toastType = .info - toastModel?.toastDescription = "Search cancelled" - default: - toastModel?.toastDescription = "Error in fetching data \(error)" + Task { @MainActor in + switch error.code { + case -999: + toastModel?.toastType = .info + toastModel?.toastDescription = "Search cancelled" + default: + toastModel?.toastDescription = "Error in fetching data \(error)" + } } - print("Error in fetching data \(error)") return nil } } + public func scrapeJson(source: Source, jsonData: Data) -> [SearchResult] { + var tempResults: [SearchResult] = [] + + guard let jsonParser = source.jsonParser else { + return tempResults + } + + var jsonResults: [JSON] = [] + + do { + let json = try JSON(data: jsonData) + + if let resultsQuery = jsonParser.results { + jsonResults = json[resultsQuery.components(separatedBy: ".")].arrayValue + } else { + jsonResults = json.arrayValue + } + } catch { + if let api = source.api { + Task { @MainActor in + cleanApiCreds(api: api) + } + + print("JSON parsing error, couldn't fetch results: \(error)") + } + } + + // If there are no results and the client secret isn't dynamic, just clear out the token + if let api = source.api, jsonResults.isEmpty { + Task { @MainActor in + cleanApiCreds(api: api) + } + + print("JSON results were empty!") + } + + // Iterate through results and grab what we can + for result in jsonResults { + var subResults: [JSON] = [] + + let searchResult = parseJsonResult(result, jsonParser: jsonParser, source: source) + + // If subresults exist, iterate through those as well with the existing result + // Otherwise append the applied result if it exists + // Better to be redundant with checks rather than another for loop or filter + if let subResultsQuery = jsonParser.subResults { + subResults = result[subResultsQuery.components(separatedBy: ".")].arrayValue + + for subResult in subResults { + if let newSearchResult = + parseJsonResult( + subResult, + jsonParser: jsonParser, + source: source, + existingSearchResult: searchResult + ), + let magnetLink = newSearchResult.magnetLink, + magnetLink.starts(with: "magnet:"), + !tempResults.contains(newSearchResult) + { + tempResults.append(newSearchResult) + } + } + } else if + let searchResult = searchResult, + let magnetLink = searchResult.magnetLink, + magnetLink.starts(with: "magnet:"), + !tempResults.contains(searchResult) + { + tempResults.append(searchResult) + } + } + + return tempResults + } + + public func parseJsonResult(_ result: JSON, jsonParser: SourceJsonParser, source: Source, existingSearchResult: SearchResult? = nil) -> SearchResult? { + var magnetHash: String? = existingSearchResult?.magnetHash + + if let magnetHashParser = jsonParser.magnetHash { + let rawHash = result[magnetHashParser.query.components(separatedBy: ".")].rawValue + + if !(rawHash is NSNull) { + magnetHash = fetchMagnetHash(existingHash: String(describing: rawHash)) + } + } + + var title: String? = existingSearchResult?.title + + if let titleParser = jsonParser.title { + if let existingTitle = existingSearchResult?.title, + let discriminatorQuery = titleParser.discriminator + { + let rawDiscriminator = result[discriminatorQuery.components(separatedBy: ".")].rawValue + + if !(rawDiscriminator is NSNull) { + title = String(describing: rawDiscriminator) + existingTitle + } + } else if existingSearchResult?.title == nil { + let rawTitle = result[titleParser.query].rawValue + title = rawTitle is NSNull ? nil : String(describing: rawTitle) + } + } + + var link: String? = existingSearchResult?.magnetLink + + if let magnetLinkParser = jsonParser.magnetLink, existingSearchResult?.magnetLink == nil { + let rawLink = result[magnetLinkParser.query.components(separatedBy: ".")].rawValue + link = rawLink is NSNull ? nil : String(describing: rawLink) + } else if let magnetHash = magnetHash { + link = generateMagnetLink(magnetHash: magnetHash, title: title, trackers: source.trackers) + } + + if magnetHash == nil, let href = link { + magnetHash = fetchMagnetHash(magnetLink: href) + } + + var size: String? = existingSearchResult?.size + + if let sizeParser = jsonParser.size, existingSearchResult?.size == nil { + let rawSize = result[sizeParser.query.components(separatedBy: ".")].rawValue + size = rawSize is NSNull ? nil : String(describing: rawSize) + } + + if let sizeString = size, let sizeInt = Int64(sizeString) { + size = byteCountFormatter.string(fromByteCount: sizeInt) + } + + var seeders: String? = existingSearchResult?.seeders + var leechers: String? = existingSearchResult?.leechers + + if let seederLeecher = jsonParser.seedLeech { + if let seederQuery = seederLeecher.seeders, existingSearchResult?.seeders == nil { + let rawSeeders = result[seederQuery.components(separatedBy: ".")].rawValue + seeders = rawSeeders is NSNull ? nil : String(describing: rawSeeders) + } + + if let leecherQuery = seederLeecher.leechers, existingSearchResult?.leechers == nil { + let rawLeechers = result[leecherQuery.components(separatedBy: ".")].rawValue + leechers = rawLeechers is NSNull ? nil : String(describing: rawLeechers) + } + } + + let result = SearchResult( + title: title, + source: source.name, + size: size, + magnetLink: link, + magnetHash: magnetHash, + seeders: seeders, + leechers: leechers + ) + + return result + } + // RSS feed scraper - @MainActor public func scrapeRss(source: Source, rss: String) -> [SearchResult] { var tempResults: [SearchResult] = [] @@ -171,7 +432,9 @@ class ScrapingViewModel: ObservableObject { let document = try SwiftSoup.parse(rss, "", Parser.xmlParser()) items = try document.getElementsByTag("item") } catch { - toastModel?.toastDescription = "RSS scraping error, couldn't fetch items: \(error)" + Task { @MainActor in + toastModel?.toastDescription = "RSS scraping error, couldn't fetch items: \(error)" + } print("RSS scraping error, couldn't fetch items: \(error)") return tempResults @@ -181,13 +444,15 @@ class ScrapingViewModel: ObservableObject { // Parse magnet link or translate hash var magnetHash: String? if let magnetHashParser = rssParser.magnetHash { - magnetHash = try? runRssComplexQuery( + let tempHash = try? runRssComplexQuery( item: item, query: magnetHashParser.query, attribute: magnetHashParser.attribute, - lookupAttribute: magnetHashParser.lookupAttribute, + discriminator: magnetHashParser.discriminator, regexString: magnetHashParser.regex ) + + magnetHash = fetchMagnetHash(existingHash: tempHash) } var title: String? @@ -196,7 +461,7 @@ class ScrapingViewModel: ObservableObject { item: item, query: titleParser.query, attribute: titleParser.attribute, - lookupAttribute: titleParser.lookupAttribute, + discriminator: titleParser.discriminator, regexString: titleParser.regex ) } @@ -207,11 +472,11 @@ class ScrapingViewModel: ObservableObject { item: item, query: magnetLinkParser.query, attribute: magnetLinkParser.attribute, - lookupAttribute: magnetLinkParser.lookupAttribute, + discriminator: magnetLinkParser.discriminator, regexString: magnetLinkParser.regex ) } else if let magnetHash = magnetHash { - link = generateMagnetLink(magnetHash: magnetHash, title: title, trackers: rssParser.trackers) + link = generateMagnetLink(magnetHash: magnetHash, title: title, trackers: source.trackers) } else { continue } @@ -230,7 +495,7 @@ class ScrapingViewModel: ObservableObject { item: item, query: sizeParser.query, attribute: sizeParser.attribute, - lookupAttribute: sizeParser.lookupAttribute, + discriminator: sizeParser.discriminator, regexString: sizeParser.regex ) } @@ -247,7 +512,7 @@ class ScrapingViewModel: ObservableObject { item: item, query: seederQuery, attribute: seederLeecher.attribute, - lookupAttribute: seederLeecher.lookupAttribute, + discriminator: seederLeecher.discriminator, regexString: seederLeecher.seederRegex ) } @@ -257,7 +522,7 @@ class ScrapingViewModel: ObservableObject { item: item, query: leecherQuery, attribute: seederLeecher.attribute, - lookupAttribute: seederLeecher.lookupAttribute, + discriminator: seederLeecher.discriminator, regexString: seederLeecher.leecherRegex ) } @@ -281,8 +546,36 @@ class ScrapingViewModel: ObservableObject { return tempResults } + // Complex query parsing for RSS scraping + func runRssComplexQuery(item: Element, query: String, attribute: String, discriminator: String?, regexString: String?) throws -> String? { + var parsedValue: String? + + switch attribute { + case "text": + parsedValue = try item.getElementsByTag(query).first()?.text() + default: + // If there's a key/value to lookup the attribute with, query it. Othewise assume the value is in the same attribute + if let discriminator = discriminator { + let containerElement = try item.getElementsByAttributeValue(discriminator, query).first() + parsedValue = try containerElement?.attr(attribute) + } else { + let containerElement = try item.getElementsByAttribute(attribute).first() + parsedValue = try containerElement?.attr(attribute) + } + } + + // A capture group must be used in the provided regex + if let regexString = regexString, + let parsedValue = parsedValue, + let regexValue = try? Regex(regexString).firstMatch(in: parsedValue)?.groups[safe: 0]?.value + { + return regexValue + } else { + return parsedValue + } + } + // HTML scraper - @MainActor public func scrapeHtml(source: Source, baseUrl: String, html: String) async -> [SearchResult] { var tempResults: [SearchResult] = [] @@ -296,7 +589,9 @@ class ScrapingViewModel: ObservableObject { let document = try SwiftSoup.parse(html) rows = try document.select(htmlParser.rows) } catch { - toastModel?.toastDescription = "Scraping error, couldn't fetch rows: \(error)" + Task { @MainActor in + toastModel?.toastDescription = "Scraping error, couldn't fetch rows: \(error)" + } print("Scraping error, couldn't fetch rows: \(error)") return tempResults @@ -314,11 +609,11 @@ class ScrapingViewModel: ObservableObject { var href: String if let externalMagnetQuery = magnetParser.externalLinkQuery, !externalMagnetQuery.isEmpty { - guard let externalMagnetLink = try row.select(externalMagnetQuery).first()?.attr("href") else { - continue - } - - guard let magnetHtml = await fetchWebsiteData(urlString: baseUrl + externalMagnetLink) else { + guard + let externalMagnetLink = try row.select(externalMagnetQuery).first()?.attr("href"), + let data = await fetchWebsiteData(urlString: baseUrl + externalMagnetLink), + let magnetHtml = String(data: data, encoding: .utf8) + else { continue } @@ -429,7 +724,9 @@ class ScrapingViewModel: ObservableObject { tempResults.append(result) } } catch { - toastModel?.toastDescription = "Scraping error: \(error)" + Task { @MainActor in + toastModel?.toastDescription = "Scraping error: \(error)" + } print("Scraping error: \(error)") continue @@ -463,42 +760,19 @@ class ScrapingViewModel: ObservableObject { } } - // Complex query parsing for RSS scraping - func runRssComplexQuery(item: Element, query: String, attribute: String, lookupAttribute: String?, regexString: String?) throws -> String? { - var parsedValue: String? - - switch attribute { - case "text": - parsedValue = try item.getElementsByTag(query).first()?.text() - default: - // If there's a key/value to lookup the attribute with, query it. Othewise assume the value is in the same attribute - if let lookupAttribute = lookupAttribute { - let containerElement = try item.getElementsByAttributeValue(lookupAttribute, query).first() - parsedValue = try containerElement?.attr(attribute) - } else { - let containerElement = try item.getElementsByAttribute(attribute).first() - parsedValue = try containerElement?.attr(attribute) - } - } - - // A capture group must be used in the provided regex - if let regexString = regexString, - let parsedValue = parsedValue, - let regexValue = try? Regex(regexString).firstMatch(in: parsedValue)?.groups[safe: 0]?.value - { - return regexValue - } else { - return parsedValue - } - } - // Fetches and possibly converts the magnet hash value to sha1 - public func fetchMagnetHash(magnetLink: String) -> String? { - guard let firstSplit = magnetLink.split(separator: ":")[safe: 3] else { - return nil - } + public func fetchMagnetHash(magnetLink: String? = nil, existingHash: String? = nil) -> String? { + var magnetHash: String - guard let magnetHash = firstSplit.split(separator: "&")[safe: 0] else { + if let existingHash = existingHash { + magnetHash = existingHash + } else if + let magnetLink = magnetLink, + let firstSplit = magnetLink.split(separator: ":")[safe: 3], + let tempHash = firstSplit.split(separator: "&")[safe: 0] + { + magnetHash = String(tempHash) + } else { return nil } @@ -554,4 +828,27 @@ class ScrapingViewModel: ObservableObject { return magnetLinkArray.joined() } + + @MainActor + func cleanApiCreds(api: SourceApi) { + let backgroundContext = PersistenceController.shared.backgroundContext + + var clientIdReset = false + var clientSecretReset = false + + if let clientId = api.clientId, !clientId.dynamic { + clientId.value = nil + clientIdReset = true + } + + if let clientSecret = api.clientSecret, !clientSecret.dynamic { + clientSecret.value = nil + clientSecretReset = true + } + + toastModel?.toastDescription = + "Could not fetch results, your \(clientIdReset ? "client ID" : "") \(clientIdReset && clientSecretReset ? "and" : "") \(clientSecretReset ? "token" : "") was automatically reset. Make sure all credentials are correct in the source's settings!" + + PersistenceController.shared.save(backgroundContext) + } } diff --git a/Ferrite/ViewModels/SourceManager.swift b/Ferrite/ViewModels/SourceManager.swift index 4ea52d9..07c44fb 100644 --- a/Ferrite/ViewModels/SourceManager.swift +++ b/Ferrite/ViewModels/SourceManager.swift @@ -28,7 +28,10 @@ public class SourceManager: ObservableObject { return } - let (data, _) = try await URLSession.shared.data(for: URLRequest(url: url)) + // Always get the up-to-date source list + let request = URLRequest(url: url, cachePolicy: .reloadIgnoringLocalCacheData) + + let (data, _) = try await URLSession.shared.data(for: request) var sourceResponse = try JSONDecoder().decode(SourceListJson.self, from: data) for index in sourceResponse.sources.indices { @@ -45,6 +48,17 @@ public class SourceManager: ObservableObject { } } + // Fetches sources using the background context + public func fetchInstalledSources() -> [Source] { + let backgroundContext = PersistenceController.shared.backgroundContext + + if let sources = try? backgroundContext.fetch(Source.fetchRequest()) { + return sources.compactMap { $0 } + } else { + return [] + } + } + public func installSource(sourceJson: SourceJson, doUpsert: Bool = false) { let backgroundContext = PersistenceController.shared.backgroundContext @@ -84,11 +98,16 @@ public class SourceManager: ObservableObject { newSource.baseUrl = sourceJson.baseUrl newSource.author = sourceJson.author ?? "Unknown" newSource.listId = sourceJson.listId + newSource.trackers = sourceJson.trackers if let sourceApiJson = sourceJson.api { addSourceApi(newSource: newSource, apiJson: sourceApiJson) } + if let jsonParserJson = sourceJson.jsonParser { + addJsonParser(newSource: newSource, jsonParserJson: jsonParserJson) + } + // Adds an RSS parser if present if let rssParserJson = sourceJson.rssParser { addRssParser(newSource: newSource, rssParserJson: rssParserJson) @@ -100,7 +119,9 @@ public class SourceManager: ObservableObject { } // Add an API condition as well - if newSource.rssParser != nil { + if newSource.jsonParser != nil { + newSource.preferredParser = Int16(SourcePreferredParser.siteApi.rawValue) + } else if newSource.rssParser != nil { newSource.preferredParser = Int16(SourcePreferredParser.rss.rawValue) } else { newSource.preferredParser = Int16(SourcePreferredParser.scraping.rawValue) @@ -121,21 +142,98 @@ public class SourceManager: ObservableObject { let backgroundContext = PersistenceController.shared.backgroundContext let newSourceApi = SourceApi(context: backgroundContext) - newSourceApi.clientId = apiJson.clientId + newSourceApi.apiUrl = apiJson.apiUrl - if let clientId = apiJson.clientId { - newSourceApi.clientId = clientId + if let clientIdJson = apiJson.clientId { + let newClientId = SourceApiClientId(context: backgroundContext) + newClientId.query = clientIdJson.query + newClientId.urlString = clientIdJson.url + newClientId.dynamic = clientIdJson.dynamic ?? false + newClientId.value = clientIdJson.value + newClientId.responseType = clientIdJson.responseType?.rawValue ?? ApiCredentialResponseType.json.rawValue + newClientId.expiryLength = clientIdJson.expiryLength ?? 0 + newClientId.timeStamp = Date() + + newSourceApi.clientId = newClientId } - newSourceApi.dynamicClientId = apiJson.dynamicClientId ?? false + if let clientSecretJson = apiJson.clientSecret { + let newClientSecret = SourceApiClientSecret(context: backgroundContext) + newClientSecret.query = clientSecretJson.query + newClientSecret.urlString = clientSecretJson.url + newClientSecret.dynamic = clientSecretJson.dynamic ?? false + newClientSecret.value = clientSecretJson.value + newClientSecret.responseType = clientSecretJson.responseType?.rawValue ?? ApiCredentialResponseType.json.rawValue + newClientSecret.expiryLength = clientSecretJson.expiryLength ?? 0 + newClientSecret.timeStamp = Date() - if apiJson.usesSecret { - newSourceApi.clientSecret = "" + newSourceApi.clientSecret = newClientSecret } newSource.api = newSourceApi } + func addJsonParser(newSource: Source, jsonParserJson: SourceJsonParserJson) { + let backgroundContext = PersistenceController.shared.backgroundContext + + let newSourceJsonParser = SourceJsonParser(context: backgroundContext) + newSourceJsonParser.searchUrl = jsonParserJson.searchUrl + newSourceJsonParser.results = jsonParserJson.results + newSourceJsonParser.subResults = jsonParserJson.subResults + + // Tune these complex queries to the final JSON parser format + if let magnetLinkJson = jsonParserJson.magnetLink { + let newSourceMagnetLink = SourceMagnetLink(context: backgroundContext) + newSourceMagnetLink.query = magnetLinkJson.query + newSourceMagnetLink.attribute = magnetLinkJson.attribute ?? "text" + newSourceMagnetLink.discriminator = magnetLinkJson.discriminator + + newSourceJsonParser.magnetLink = newSourceMagnetLink + } + + if let magnetHashJson = jsonParserJson.magnetHash { + let newSourceMagnetHash = SourceMagnetHash(context: backgroundContext) + newSourceMagnetHash.query = magnetHashJson.query + newSourceMagnetHash.attribute = magnetHashJson.attribute ?? "text" + newSourceMagnetHash.discriminator = magnetHashJson.discriminator + + newSourceJsonParser.magnetHash = newSourceMagnetHash + } + + if let titleJson = jsonParserJson.title { + let newSourceTitle = SourceTitle(context: backgroundContext) + newSourceTitle.query = titleJson.query + newSourceTitle.attribute = titleJson.attribute ?? "text" + newSourceTitle.discriminator = titleJson.discriminator + + newSourceJsonParser.title = newSourceTitle + } + + if let sizeJson = jsonParserJson.size { + let newSourceSize = SourceSize(context: backgroundContext) + newSourceSize.query = sizeJson.query + newSourceSize.attribute = sizeJson.attribute ?? "text" + newSourceSize.discriminator = sizeJson.discriminator + + newSourceJsonParser.size = newSourceSize + } + + if let seedLeechJson = jsonParserJson.sl { + let newSourceSeedLeech = SourceSeedLeech(context: backgroundContext) + newSourceSeedLeech.seeders = seedLeechJson.seeders + newSourceSeedLeech.leechers = seedLeechJson.leechers + newSourceSeedLeech.combined = seedLeechJson.combined + newSourceSeedLeech.attribute = seedLeechJson.attribute ?? "text" + newSourceSeedLeech.discriminator = seedLeechJson.discriminator + newSourceSeedLeech.seederRegex = seedLeechJson.seederRegex + newSourceSeedLeech.leecherRegex = seedLeechJson.leecherRegex + + newSourceJsonParser.seedLeech = newSourceSeedLeech + } + + newSource.jsonParser = newSourceJsonParser + } + func addRssParser(newSource: Source, rssParserJson: SourceRssParserJson) { let backgroundContext = PersistenceController.shared.backgroundContext @@ -143,13 +241,12 @@ public class SourceManager: ObservableObject { newSourceRssParser.rssUrl = rssParserJson.rssUrl newSourceRssParser.searchUrl = rssParserJson.searchUrl newSourceRssParser.items = rssParserJson.items - newSourceRssParser.trackers = rssParserJson.trackers if let magnetLinkJson = rssParserJson.magnetLink { let newSourceMagnetLink = SourceMagnetLink(context: backgroundContext) newSourceMagnetLink.query = magnetLinkJson.query newSourceMagnetLink.attribute = magnetLinkJson.attribute ?? "text" - newSourceMagnetLink.lookupAttribute = magnetLinkJson.lookupAttribute + newSourceMagnetLink.discriminator = magnetLinkJson.discriminator newSourceRssParser.magnetLink = newSourceMagnetLink } @@ -158,7 +255,7 @@ public class SourceManager: ObservableObject { let newSourceMagnetHash = SourceMagnetHash(context: backgroundContext) newSourceMagnetHash.query = magnetHashJson.query newSourceMagnetHash.attribute = magnetHashJson.attribute ?? "text" - newSourceMagnetHash.lookupAttribute = magnetHashJson.lookupAttribute + newSourceMagnetHash.discriminator = magnetHashJson.discriminator newSourceRssParser.magnetHash = newSourceMagnetHash } @@ -167,7 +264,7 @@ public class SourceManager: ObservableObject { let newSourceTitle = SourceTitle(context: backgroundContext) newSourceTitle.query = titleJson.query newSourceTitle.attribute = titleJson.attribute ?? "text" - newSourceTitle.lookupAttribute = newSourceTitle.lookupAttribute + newSourceTitle.discriminator = titleJson.discriminator newSourceRssParser.title = newSourceTitle } @@ -176,7 +273,7 @@ public class SourceManager: ObservableObject { let newSourceSize = SourceSize(context: backgroundContext) newSourceSize.query = sizeJson.query newSourceSize.attribute = sizeJson.attribute ?? "text" - newSourceSize.lookupAttribute = sizeJson.lookupAttribute + newSourceSize.discriminator = sizeJson.discriminator newSourceRssParser.size = newSourceSize } @@ -187,7 +284,7 @@ public class SourceManager: ObservableObject { newSourceSeedLeech.leechers = seedLeechJson.leechers newSourceSeedLeech.combined = seedLeechJson.combined newSourceSeedLeech.attribute = seedLeechJson.attribute ?? "text" - newSourceSeedLeech.lookupAttribute = seedLeechJson.lookupAttribute + newSourceSeedLeech.discriminator = seedLeechJson.discriminator newSourceSeedLeech.seederRegex = seedLeechJson.seederRegex newSourceSeedLeech.leecherRegex = seedLeechJson.leecherRegex diff --git a/Ferrite/Views/ContentView.swift b/Ferrite/Views/ContentView.swift index 85f0a3c..51b0ae6 100644 --- a/Ferrite/Views/ContentView.swift +++ b/Ferrite/Views/ContentView.swift @@ -12,6 +12,7 @@ struct ContentView: View { @EnvironmentObject var scrapingModel: ScrapingViewModel @EnvironmentObject var debridManager: DebridManager @EnvironmentObject var navModel: NavigationViewModel + @EnvironmentObject var sourceManager: SourceManager @AppStorage("RealDebrid.Enabled") var realDebridEnabled = false @@ -98,13 +99,17 @@ struct ContentView: View { } .navigationTitle("Search") .navigationSearchBar { - SearchBar("Search", text: $scrapingModel.searchText, isEditing: $navModel.isEditingSearch, + SearchBar("Search", + text: $scrapingModel.searchText, + isEditing: $navModel.isEditingSearch, onCommit: { + scrapingModel.searchResults = [] scrapingModel.runningSearchTask = Task { navModel.isSearching = true navModel.showSearchProgress = true - await scrapingModel.scanSources(sources: sources.compactMap { $0 }) + let sources = sourceManager.fetchInstalledSources() + await scrapingModel.scanSources(sources: sources) if realDebridEnabled, !scrapingModel.searchResults.isEmpty { await debridManager.populateDebridHashes(scrapingModel.searchResults) diff --git a/Ferrite/Views/MagnetChoiceView.swift b/Ferrite/Views/MagnetChoiceView.swift index a71bb66..b8da57e 100644 --- a/Ferrite/Views/MagnetChoiceView.swift +++ b/Ferrite/Views/MagnetChoiceView.swift @@ -74,7 +74,10 @@ struct MagnetChoiceView: View { } ListRowButtonView("Share magnet", systemImage: "square.and.arrow.up.fill") { - if let result = scrapingModel.selectedSearchResult, let url = URL(string: result.magnetLink) { + if let result = scrapingModel.selectedSearchResult, + let magnetLink = result.magnetLink, + let url = URL(string: magnetLink) + { activityItems = [url] navModel.showActivityView.toggle() } diff --git a/Ferrite/Views/SearchResultRDView.swift b/Ferrite/Views/SearchResultRDView.swift index f422a7c..dbcc913 100644 --- a/Ferrite/Views/SearchResultRDView.swift +++ b/Ferrite/Views/SearchResultRDView.swift @@ -28,7 +28,9 @@ struct SearchResultRDView: View { Text("L: \(leechers)") } - Text(result.size) + if let size = result.size { + Text(size) + } if realDebridEnabled { Text("RD") diff --git a/Ferrite/Views/SearchResultsView.swift b/Ferrite/Views/SearchResultsView.swift index 5ded689..943292a 100644 --- a/Ferrite/Views/SearchResultsView.swift +++ b/Ferrite/Views/SearchResultsView.swift @@ -39,7 +39,7 @@ struct SearchResultsView: View { navModel.runMagnetAction(action: nil, searchResult: result) } } label: { - Text(result.title) + Text(result.title ?? "No title") .font(.callout) .fixedSize(horizontal: false, vertical: true) } diff --git a/Ferrite/Views/SourceViews/SourceSettingsView.swift b/Ferrite/Views/SourceViews/SourceSettingsView.swift index 4a6a8a6..ccf6456 100644 --- a/Ferrite/Views/SourceViews/SourceSettingsView.swift +++ b/Ferrite/Views/SourceViews/SourceSettingsView.swift @@ -46,7 +46,9 @@ struct SourceSettingsView: View { SourceSettingsBaseUrlView(selectedSource: selectedSource) } - if let sourceApi = selectedSource.api { + if let sourceApi = selectedSource.api, + sourceApi.clientId?.dynamic ?? false || sourceApi.clientSecret?.dynamic ?? false + { SourceSettingsApiView(selectedSourceApi: sourceApi) } @@ -110,27 +112,29 @@ struct SourceSettingsApiView: View { header: Text("API credentials"), footer: Text("Grab the required API credentials from the website. A client secret can be an API token.") ) { - if selectedSourceApi.dynamicClientId { + if let clientId = selectedSourceApi.clientId, clientId.dynamic { TextField("Client ID", text: $tempClientId, onEditingChanged: { isFocused in if !isFocused { - selectedSourceApi.clientId = tempClientId + clientId.value = tempClientId + clientId.timeStamp = Date() } }) .autocapitalization(.none) .onAppear { - tempClientId = selectedSourceApi.clientId ?? "" + tempClientId = clientId.value ?? "" } } - if selectedSourceApi.clientSecret != nil { + if let clientSecret = selectedSourceApi.clientSecret, clientSecret.dynamic { TextField("Token", text: $tempClientSecret, onEditingChanged: { isFocused in if !isFocused { - selectedSourceApi.clientSecret = tempClientSecret + clientSecret.value = tempClientSecret + clientSecret.timeStamp = Date() } }) .autocapitalization(.none) .onAppear { - tempClientSecret = selectedSourceApi.clientSecret ?? "" + tempClientSecret = clientSecret.value ?? "" } } } @@ -144,15 +148,20 @@ struct SourceSettingsMethodView: View { var body: some View { Picker("Fetch method", selection: $selectedTempParser) { - if selectedSource.htmlParser != nil { - Text("Web scraping") - .tag(SourcePreferredParser.scraping) + if selectedSource.jsonParser != nil { + Text("Website API") + .tag(SourcePreferredParser.siteApi) } if selectedSource.rssParser != nil { Text("RSS") .tag(SourcePreferredParser.rss) } + + if selectedSource.htmlParser != nil { + Text("Web scraping") + .tag(SourcePreferredParser.scraping) + } } .pickerStyle(.inline) .onAppear {