From 42e202b207a7c073f421a48c3a8903c1d5c1e59e Mon Sep 17 00:00:00 2001 From: kingbri Date: Mon, 10 Jun 2024 23:35:17 -0400 Subject: [PATCH] Plugins: Add request options to sources Adds HTTP method, headers, and a body string. Also use a common function to substitute params rather to allow for maintanence of a common dictionary. Signed-off-by: kingbri --- Ferrite.xcodeproj/project.pbxproj | 8 +++ .../SourceHtmlParser+CoreDataProperties.swift | 1 + .../SourceJsonParser+CoreDataProperties.swift | 1 + .../Classes/SourceRequest+CoreDataClass.swift | 13 ++++ .../SourceRequest+CoreDataProperties.swift | 25 +++++++ .../SourceRssParser+CoreDataProperties.swift | 1 + .../FerriteDB_v2.xcdatamodel/contents | 13 +++- Ferrite/Models/SourceModels.swift | 9 +++ Ferrite/ViewModels/PluginManager.swift | 25 +++++++ Ferrite/ViewModels/ScrapingViewModel.swift | 72 ++++++++++++++----- 10 files changed, 148 insertions(+), 20 deletions(-) create mode 100644 Ferrite/DataManagement/Classes/SourceRequest+CoreDataClass.swift create mode 100644 Ferrite/DataManagement/Classes/SourceRequest+CoreDataProperties.swift diff --git a/Ferrite.xcodeproj/project.pbxproj b/Ferrite.xcodeproj/project.pbxproj index 3c2c2bd..3763f2d 100644 --- a/Ferrite.xcodeproj/project.pbxproj +++ b/Ferrite.xcodeproj/project.pbxproj @@ -154,6 +154,8 @@ 0CE1C4182981E8D700418F20 /* Plugin.swift in Sources */ = {isa = PBXBuildFile; fileRef = 0CE1C4172981E8D700418F20 /* Plugin.swift */; }; 0CEC8AAE299B31B6007BFE8F /* SearchFilterHeaderView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 0CEC8AAD299B31B6007BFE8F /* SearchFilterHeaderView.swift */; }; 0CEC8AB2299B3B57007BFE8F /* LibraryPickerView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 0CEC8AB1299B3B57007BFE8F /* LibraryPickerView.swift */; }; + 0CEE11B12C1743E0004C8BB2 /* SourceRequest+CoreDataClass.swift in Sources */ = {isa = PBXBuildFile; fileRef = 0CEE11AF2C1743E0004C8BB2 /* SourceRequest+CoreDataClass.swift */; }; + 0CEE11B22C1743E0004C8BB2 /* SourceRequest+CoreDataProperties.swift in Sources */ = {isa = PBXBuildFile; fileRef = 0CEE11B02C1743E0004C8BB2 /* SourceRequest+CoreDataProperties.swift */; }; 0CF1ABDC2C0C04B2009F6C26 /* Debrid.swift in Sources */ = {isa = PBXBuildFile; fileRef = 0CF1ABDB2C0C04B2009F6C26 /* Debrid.swift */; }; 0CF1ABE22C0C3D2F009F6C26 /* DebridModels.swift in Sources */ = {isa = PBXBuildFile; fileRef = 0CF1ABE12C0C3D2F009F6C26 /* DebridModels.swift */; }; 0CF2C0A529D1EBD400E716DD /* UIApplication.swift in Sources */ = {isa = PBXBuildFile; fileRef = 0CF2C0A429D1EBD400E716DD /* UIApplication.swift */; }; @@ -302,6 +304,8 @@ 0CE1C4172981E8D700418F20 /* Plugin.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Plugin.swift; sourceTree = ""; }; 0CEC8AAD299B31B6007BFE8F /* SearchFilterHeaderView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SearchFilterHeaderView.swift; sourceTree = ""; }; 0CEC8AB1299B3B57007BFE8F /* LibraryPickerView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = LibraryPickerView.swift; sourceTree = ""; }; + 0CEE11AF2C1743E0004C8BB2 /* SourceRequest+CoreDataClass.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "SourceRequest+CoreDataClass.swift"; sourceTree = ""; }; + 0CEE11B02C1743E0004C8BB2 /* SourceRequest+CoreDataProperties.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "SourceRequest+CoreDataProperties.swift"; sourceTree = ""; }; 0CF1ABDB2C0C04B2009F6C26 /* Debrid.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Debrid.swift; sourceTree = ""; }; 0CF1ABE12C0C3D2F009F6C26 /* DebridModels.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DebridModels.swift; sourceTree = ""; }; 0CF2C0A429D1EBD400E716DD /* UIApplication.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = UIApplication.swift; sourceTree = ""; }; @@ -383,6 +387,8 @@ 0C84F47B2895BFED0074B7C9 /* Source+CoreDataProperties.swift */, 0C84F47C2895BFED0074B7C9 /* SourceHtmlParser+CoreDataClass.swift */, 0C84F47D2895BFED0074B7C9 /* SourceHtmlParser+CoreDataProperties.swift */, + 0CEE11AF2C1743E0004C8BB2 /* SourceRequest+CoreDataClass.swift */, + 0CEE11B02C1743E0004C8BB2 /* SourceRequest+CoreDataProperties.swift */, ); path = Classes; sourceTree = ""; @@ -927,6 +933,8 @@ 0C50055B2992BA6A0064606A /* PluginTag+CoreDataProperties.swift in Sources */, 0C7075E629D3845D0093DB2D /* ShareSheet.swift in Sources */, 0C871BDF29994D9D005279AC /* FilterLabelView.swift in Sources */, + 0CEE11B12C1743E0004C8BB2 /* SourceRequest+CoreDataClass.swift in Sources */, + 0CEE11B22C1743E0004C8BB2 /* SourceRequest+CoreDataProperties.swift in Sources */, 0CC2CA7429E24F63000A8585 /* ExpandedSearchable.swift in Sources */, 0CF1ABE22C0C3D2F009F6C26 /* DebridModels.swift in Sources */, 0C6771F429B3B4FD005D38D2 /* KodiWrapper.swift in Sources */, diff --git a/Ferrite/DataManagement/Classes/SourceHtmlParser+CoreDataProperties.swift b/Ferrite/DataManagement/Classes/SourceHtmlParser+CoreDataProperties.swift index 11f9202..edc0c77 100644 --- a/Ferrite/DataManagement/Classes/SourceHtmlParser+CoreDataProperties.swift +++ b/Ferrite/DataManagement/Classes/SourceHtmlParser+CoreDataProperties.swift @@ -16,6 +16,7 @@ public extension SourceHtmlParser { @NSManaged var rows: String @NSManaged var searchUrl: String? + @NSManaged var request: SourceRequest? @NSManaged var magnetHash: SourceMagnetHash? @NSManaged var magnetLink: SourceMagnetLink? @NSManaged var parentSource: Source? diff --git a/Ferrite/DataManagement/Classes/SourceJsonParser+CoreDataProperties.swift b/Ferrite/DataManagement/Classes/SourceJsonParser+CoreDataProperties.swift index 81d7287..bb52c0a 100644 --- a/Ferrite/DataManagement/Classes/SourceJsonParser+CoreDataProperties.swift +++ b/Ferrite/DataManagement/Classes/SourceJsonParser+CoreDataProperties.swift @@ -17,6 +17,7 @@ public extension SourceJsonParser { @NSManaged var results: String? @NSManaged var subResults: String? @NSManaged var searchUrl: String + @NSManaged var request: SourceRequest? @NSManaged var magnetHash: SourceMagnetHash? @NSManaged var magnetLink: SourceMagnetLink? @NSManaged var parentSource: Source? diff --git a/Ferrite/DataManagement/Classes/SourceRequest+CoreDataClass.swift b/Ferrite/DataManagement/Classes/SourceRequest+CoreDataClass.swift new file mode 100644 index 0000000..18d7bf4 --- /dev/null +++ b/Ferrite/DataManagement/Classes/SourceRequest+CoreDataClass.swift @@ -0,0 +1,13 @@ +// +// SourceRequest+CoreDataClass.swift +// Ferrite +// +// Created by Brian Dashore on 6/10/24. +// +// + +import CoreData +import Foundation + +@objc(SourceRequest) +public class SourceRequest: NSManagedObject {} diff --git a/Ferrite/DataManagement/Classes/SourceRequest+CoreDataProperties.swift b/Ferrite/DataManagement/Classes/SourceRequest+CoreDataProperties.swift new file mode 100644 index 0000000..decdb10 --- /dev/null +++ b/Ferrite/DataManagement/Classes/SourceRequest+CoreDataProperties.swift @@ -0,0 +1,25 @@ +// +// SourceRequest+CoreDataProperties.swift +// Ferrite +// +// Created by Brian Dashore on 6/10/24. +// +// + +import CoreData +import Foundation + +public extension SourceRequest { + @nonobjc class func fetchRequest() -> NSFetchRequest { + NSFetchRequest(entityName: "SourceRequest") + } + + @NSManaged var method: String? + @NSManaged var headers: [String: String]? + @NSManaged var body: String? + @NSManaged var parentHtmlParser: SourceHtmlParser? + @NSManaged var parentRssParser: SourceRssParser? + @NSManaged var parentJsonParser: SourceJsonParser? +} + +extension SourceRequest: Identifiable {} diff --git a/Ferrite/DataManagement/Classes/SourceRssParser+CoreDataProperties.swift b/Ferrite/DataManagement/Classes/SourceRssParser+CoreDataProperties.swift index 6e8cc8e..743d2cc 100644 --- a/Ferrite/DataManagement/Classes/SourceRssParser+CoreDataProperties.swift +++ b/Ferrite/DataManagement/Classes/SourceRssParser+CoreDataProperties.swift @@ -17,6 +17,7 @@ public extension SourceRssParser { @NSManaged var items: String @NSManaged var rssUrl: String? @NSManaged var searchUrl: String + @NSManaged var request: SourceRequest? @NSManaged var magnetHash: SourceMagnetHash? @NSManaged var magnetLink: SourceMagnetLink? @NSManaged var parentSource: Source? diff --git a/Ferrite/DataManagement/FerriteDB.xcdatamodeld/FerriteDB_v2.xcdatamodel/contents b/Ferrite/DataManagement/FerriteDB.xcdatamodeld/FerriteDB_v2.xcdatamodel/contents index eb3bb9c..0b4a0ce 100644 --- a/Ferrite/DataManagement/FerriteDB.xcdatamodeld/FerriteDB_v2.xcdatamodel/contents +++ b/Ferrite/DataManagement/FerriteDB.xcdatamodeld/FerriteDB_v2.xcdatamodel/contents @@ -1,5 +1,5 @@ - + @@ -106,6 +106,7 @@ + @@ -118,6 +119,7 @@ + @@ -134,6 +136,14 @@ + + + + + + + + @@ -141,6 +151,7 @@ + diff --git a/Ferrite/Models/SourceModels.swift b/Ferrite/Models/SourceModels.swift index b0f7d91..efa0efb 100644 --- a/Ferrite/Models/SourceModels.swift +++ b/Ferrite/Models/SourceModels.swift @@ -62,6 +62,7 @@ public struct SourceApiCredentialJson: Codable, Hashable, Sendable { public struct SourceJsonParserJson: Codable, Hashable, Sendable { let searchUrl: String + let request: SourceRequestJson? let results: String? let subResults: String? let title: SourceComplexQueryJson @@ -75,6 +76,7 @@ public struct SourceJsonParserJson: Codable, Hashable, Sendable { public struct SourceRssParserJson: Codable, Hashable, Sendable { let rssUrl: String? let searchUrl: String + let request: SourceRequestJson? let items: String let title: SourceComplexQueryJson let magnetHash: SourceComplexQueryJson? @@ -86,6 +88,7 @@ public struct SourceRssParserJson: Codable, Hashable, Sendable { public struct SourceHtmlParserJson: Codable, Hashable, Sendable { let searchUrl: String? + let request: SourceRequestJson? let rows: String let title: SourceComplexQueryJson let magnet: SourceMagnetJson @@ -117,3 +120,9 @@ public struct SourceSLJson: Codable, Hashable, Sendable { let seederRegex: String? let leecherRegex: String? } + +public struct SourceRequestJson: Codable, Hashable, Sendable { + let method: String? + let headers: [String: String]? + let body: String? +} diff --git a/Ferrite/ViewModels/PluginManager.swift b/Ferrite/ViewModels/PluginManager.swift index 664fdb9..bf46181 100644 --- a/Ferrite/ViewModels/PluginManager.swift +++ b/Ferrite/ViewModels/PluginManager.swift @@ -570,6 +570,7 @@ public class PluginManager: ObservableObject { newSource.api = newSourceApi } + // TODO: Migrate parser addition to a common protocol func addJsonParser(newSource: Source, jsonParserJson: SourceJsonParserJson) { let backgroundContext = PersistenceController.shared.backgroundContext @@ -578,6 +579,13 @@ public class PluginManager: ObservableObject { newSourceJsonParser.results = jsonParserJson.results newSourceJsonParser.subResults = jsonParserJson.subResults + if let requestJson = newSourceJsonParser.request { + let newParserRequest = SourceRequest(context: backgroundContext) + newParserRequest.method = requestJson.method + newParserRequest.headers = requestJson.headers + newParserRequest.body = requestJson.body + } + // Tune these complex queries to the final JSON parser format if let magnetLinkJson = jsonParserJson.magnetLink { let newSourceMagnetLink = SourceMagnetLink(context: backgroundContext) @@ -646,6 +654,13 @@ public class PluginManager: ObservableObject { newSourceRssParser.searchUrl = rssParserJson.searchUrl newSourceRssParser.items = rssParserJson.items + if let requestJson = newSourceRssParser.request { + let newParserRequest = SourceRequest(context: backgroundContext) + newParserRequest.method = requestJson.method + newParserRequest.headers = requestJson.headers + newParserRequest.body = requestJson.body + } + if let magnetLinkJson = rssParserJson.magnetLink { let newSourceMagnetLink = SourceMagnetLink(context: backgroundContext) newSourceMagnetLink.query = magnetLinkJson.query @@ -726,6 +741,16 @@ public class PluginManager: ObservableObject { newSourceHtmlParser.subName = newSourceSubName } + if let requestJson = htmlParserJson.request { + print(requestJson) + let newParserRequest = SourceRequest(context: backgroundContext) + newParserRequest.method = requestJson.method + newParserRequest.headers = requestJson.headers + newParserRequest.body = requestJson.body + + newSourceHtmlParser.request = newParserRequest + } + // Adds a title complex query let newSourceTitle = SourceTitle(context: backgroundContext) newSourceTitle.query = htmlParserJson.title.query diff --git a/Ferrite/ViewModels/ScrapingViewModel.swift b/Ferrite/ViewModels/ScrapingViewModel.swift index afdde3d..7dff35e 100644 --- a/Ferrite/ViewModels/ScrapingViewModel.swift +++ b/Ferrite/ViewModels/ScrapingViewModel.swift @@ -66,6 +66,28 @@ class ScrapingViewModel: ObservableObject { await logManager?.error(description, showToast: false) } + // Substitutes the given string with an arbitrary parameter dictionary + func substituteParams(_ input: String, with params: [String: String]) -> String { + let replaced = params.reduce(input) { result, param -> String in + result.replacingOccurrences(of: "{\(param.key)}", with: param.value) + } + + return replaced + } + + // Cleans a SourceRequest's body and headers to be substituted + func cleanRequest(request: SourceRequest, params: [String: String]) -> SourceRequest { + if let body = request.body { + request.body = substituteParams(body, with: params) + } + + if let headers = request.headers { + request.headers = headers.mapValues { substituteParams($0, with: params) } + } + + return request + } + public func scanSources(sources: [Source], searchText: String, debridManager: DebridManager) async { await logManager?.info("Started scanning sources for query \"\(searchText)\"") @@ -160,19 +182,25 @@ class ScrapingViewModel: ObservableObject { return nil } + // Initial params dict to reference + // More params are added here as needed + var params: [String: String] = [ + "query": encodedQuery + ] + switch preferredParser { case .scraping: if let htmlParser = source.htmlParser { let replacedSearchUrl = htmlParser.searchUrl.map { - $0 - .replacingOccurrences(of: "{query}", with: encodedQuery) + substituteParams($0, with: params) } let data = await handleUrls( website: website, replacedSearchUrl: replacedSearchUrl, fallbackUrls: source.fallbackUrls, - sourceName: source.name + sourceName: source.name, + requestParams: htmlParser.request.map { cleanRequest(request: $0, params: params) } ) if let data, @@ -183,23 +211,25 @@ class ScrapingViewModel: ObservableObject { } case .rss: if let rssParser = source.rssParser { - let replacedSearchUrl = rssParser.searchUrl - .replacingOccurrences(of: "{secret}", with: source.api?.clientSecret?.value ?? "") - .replacingOccurrences(of: "{query}", with: encodedQuery) + params.updateValue(source.api?.clientSecret?.value ?? "", forKey: "secret") + + let replacedSearchUrl = substituteParams(rssParser.searchUrl, with: params) // Do not use fallback URLs if the base URL isn't used let data: Data? if let rssUrl = rssParser.rssUrl { data = await fetchWebsiteData( urlString: rssUrl + replacedSearchUrl, - sourceName: source.name + sourceName: source.name, + requestParams: rssParser.request ) } else { data = await handleUrls( website: website, replacedSearchUrl: replacedSearchUrl, fallbackUrls: source.fallbackUrls, - sourceName: source.name + sourceName: source.name, + requestParams: rssParser.request ) } @@ -211,8 +241,7 @@ class ScrapingViewModel: ObservableObject { } case .siteApi: if let jsonParser = source.jsonParser { - var replacedSearchUrl = jsonParser.searchUrl - .replacingOccurrences(of: "{query}", with: encodedQuery) + var replacedSearchUrl = substituteParams(jsonParser.searchUrl, with: params) // Handle anything API related including tokens, client IDs, and appending the API URL // The source API key is for APIs that require extra credentials or use a different URL @@ -248,7 +277,8 @@ class ScrapingViewModel: ObservableObject { website: passedUrl, replacedSearchUrl: replacedSearchUrl, fallbackUrls: source.fallbackUrls, - sourceName: source.name + sourceName: source.name, + requestParams: jsonParser.request ) if let data { @@ -263,16 +293,16 @@ class ScrapingViewModel: ObservableObject { } // Checks the base URL for any website data then iterates through the fallback URLs - func handleUrls(website: String, replacedSearchUrl: String?, fallbackUrls: [String]?, sourceName: String) async -> Data? { + func handleUrls(website: String, replacedSearchUrl: String?, fallbackUrls: [String]?, sourceName: String, requestParams: SourceRequest?) async -> Data? { let fetchUrl = website + (replacedSearchUrl.map { $0 } ?? "") - if let data = await fetchWebsiteData(urlString: fetchUrl, sourceName: sourceName) { + if let data = await fetchWebsiteData(urlString: fetchUrl, sourceName: sourceName, requestParams: requestParams) { return data } if let fallbackUrls { for fallbackUrl in fallbackUrls { let fetchUrl = fallbackUrl + (replacedSearchUrl.map { $0 } ?? "") - if let data = await fetchWebsiteData(urlString: fetchUrl, sourceName: sourceName) { + if let data = await fetchWebsiteData(urlString: fetchUrl, sourceName: sourceName, requestParams: requestParams) { return data } } @@ -298,8 +328,7 @@ class ScrapingViewModel: ObservableObject { // Fetch a new credential if it's expired or doesn't exist yet if let value = credential.value, !isExpired { - return searchUrl - .replacingOccurrences(of: replacement, with: value) + return substituteParams(searchUrl, with: [replacement: value]) } else if credential.value == nil || isExpired, let credentialUrl = credential.urlString, @@ -369,7 +398,7 @@ class ScrapingViewModel: ObservableObject { } // Fetches the data for a URL - public func fetchWebsiteData(urlString: String, sourceName: String) async -> Data? { + public func fetchWebsiteData(urlString: String, sourceName: String, requestParams: SourceRequest?) async -> Data? { guard let url = URL(string: urlString.trimmingCharacters(in: .whitespacesAndNewlines)) else { await sendSourceError("\(sourceName): Source doesn't contain a valid URL, contact the source dev!") @@ -388,7 +417,12 @@ class ScrapingViewModel: ObservableObject { } } - let request = URLRequest(url: url, timeoutInterval: timeout) + var request = URLRequest(url: url, timeoutInterval: timeout) + request.httpMethod = requestParams?.method + request.httpBody = requestParams?.body?.data(using: .utf8) + requestParams?.headers?.forEach { field, value in + request.addValue(value, forHTTPHeaderField: field) + } do { let (data, _) = try await URLSession.shared.data(for: request) @@ -800,7 +834,7 @@ class ScrapingViewModel: ObservableObject { let replacedMagnetUrl = externalMagnetUrl.starts(with: "/") ? website + externalMagnetUrl : externalMagnetUrl guard - let data = await fetchWebsiteData(urlString: replacedMagnetUrl, sourceName: source.name), + let data = await fetchWebsiteData(urlString: replacedMagnetUrl, sourceName: source.name, requestParams: htmlParser.request), let magnetHtml = String(data: data, encoding: .utf8) else { continue