mirror of
https://github.com/Ferrite-iOS/Ferrite.git
synced 2026-01-11 20:10:27 +00:00
Plugins: Add request options to sources
Adds HTTP method, headers, and a body string. Also use a common function to substitute params rather to allow for maintanence of a common dictionary. Signed-off-by: kingbri <bdashore3@proton.me>
This commit is contained in:
kingbri
parent
afceea7bfb
commit
42e202b207
10 changed files with 148 additions and 20 deletions
|
|
@ -154,6 +154,8 @@
|
|||
0CE1C4182981E8D700418F20 /* Plugin.swift in Sources */ = {isa = PBXBuildFile; fileRef = 0CE1C4172981E8D700418F20 /* Plugin.swift */; };
|
||||
0CEC8AAE299B31B6007BFE8F /* SearchFilterHeaderView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 0CEC8AAD299B31B6007BFE8F /* SearchFilterHeaderView.swift */; };
|
||||
0CEC8AB2299B3B57007BFE8F /* LibraryPickerView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 0CEC8AB1299B3B57007BFE8F /* LibraryPickerView.swift */; };
|
||||
0CEE11B12C1743E0004C8BB2 /* SourceRequest+CoreDataClass.swift in Sources */ = {isa = PBXBuildFile; fileRef = 0CEE11AF2C1743E0004C8BB2 /* SourceRequest+CoreDataClass.swift */; };
|
||||
0CEE11B22C1743E0004C8BB2 /* SourceRequest+CoreDataProperties.swift in Sources */ = {isa = PBXBuildFile; fileRef = 0CEE11B02C1743E0004C8BB2 /* SourceRequest+CoreDataProperties.swift */; };
|
||||
0CF1ABDC2C0C04B2009F6C26 /* Debrid.swift in Sources */ = {isa = PBXBuildFile; fileRef = 0CF1ABDB2C0C04B2009F6C26 /* Debrid.swift */; };
|
||||
0CF1ABE22C0C3D2F009F6C26 /* DebridModels.swift in Sources */ = {isa = PBXBuildFile; fileRef = 0CF1ABE12C0C3D2F009F6C26 /* DebridModels.swift */; };
|
||||
0CF2C0A529D1EBD400E716DD /* UIApplication.swift in Sources */ = {isa = PBXBuildFile; fileRef = 0CF2C0A429D1EBD400E716DD /* UIApplication.swift */; };
|
||||
|
|
@ -302,6 +304,8 @@
|
|||
0CE1C4172981E8D700418F20 /* Plugin.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Plugin.swift; sourceTree = "<group>"; };
|
||||
0CEC8AAD299B31B6007BFE8F /* SearchFilterHeaderView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SearchFilterHeaderView.swift; sourceTree = "<group>"; };
|
||||
0CEC8AB1299B3B57007BFE8F /* LibraryPickerView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = LibraryPickerView.swift; sourceTree = "<group>"; };
|
||||
0CEE11AF2C1743E0004C8BB2 /* SourceRequest+CoreDataClass.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "SourceRequest+CoreDataClass.swift"; sourceTree = "<group>"; };
|
||||
0CEE11B02C1743E0004C8BB2 /* SourceRequest+CoreDataProperties.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "SourceRequest+CoreDataProperties.swift"; sourceTree = "<group>"; };
|
||||
0CF1ABDB2C0C04B2009F6C26 /* Debrid.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Debrid.swift; sourceTree = "<group>"; };
|
||||
0CF1ABE12C0C3D2F009F6C26 /* DebridModels.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DebridModels.swift; sourceTree = "<group>"; };
|
||||
0CF2C0A429D1EBD400E716DD /* UIApplication.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = UIApplication.swift; sourceTree = "<group>"; };
|
||||
|
|
@ -383,6 +387,8 @@
|
|||
0C84F47B2895BFED0074B7C9 /* Source+CoreDataProperties.swift */,
|
||||
0C84F47C2895BFED0074B7C9 /* SourceHtmlParser+CoreDataClass.swift */,
|
||||
0C84F47D2895BFED0074B7C9 /* SourceHtmlParser+CoreDataProperties.swift */,
|
||||
0CEE11AF2C1743E0004C8BB2 /* SourceRequest+CoreDataClass.swift */,
|
||||
0CEE11B02C1743E0004C8BB2 /* SourceRequest+CoreDataProperties.swift */,
|
||||
);
|
||||
path = Classes;
|
||||
sourceTree = "<group>";
|
||||
|
|
@ -927,6 +933,8 @@
|
|||
0C50055B2992BA6A0064606A /* PluginTag+CoreDataProperties.swift in Sources */,
|
||||
0C7075E629D3845D0093DB2D /* ShareSheet.swift in Sources */,
|
||||
0C871BDF29994D9D005279AC /* FilterLabelView.swift in Sources */,
|
||||
0CEE11B12C1743E0004C8BB2 /* SourceRequest+CoreDataClass.swift in Sources */,
|
||||
0CEE11B22C1743E0004C8BB2 /* SourceRequest+CoreDataProperties.swift in Sources */,
|
||||
0CC2CA7429E24F63000A8585 /* ExpandedSearchable.swift in Sources */,
|
||||
0CF1ABE22C0C3D2F009F6C26 /* DebridModels.swift in Sources */,
|
||||
0C6771F429B3B4FD005D38D2 /* KodiWrapper.swift in Sources */,
|
||||
|
|
|
|||
|
|
@ -16,6 +16,7 @@ public extension SourceHtmlParser {
|
|||
|
||||
@NSManaged var rows: String
|
||||
@NSManaged var searchUrl: String?
|
||||
@NSManaged var request: SourceRequest?
|
||||
@NSManaged var magnetHash: SourceMagnetHash?
|
||||
@NSManaged var magnetLink: SourceMagnetLink?
|
||||
@NSManaged var parentSource: Source?
|
||||
|
|
|
|||
|
|
@ -17,6 +17,7 @@ public extension SourceJsonParser {
|
|||
@NSManaged var results: String?
|
||||
@NSManaged var subResults: String?
|
||||
@NSManaged var searchUrl: String
|
||||
@NSManaged var request: SourceRequest?
|
||||
@NSManaged var magnetHash: SourceMagnetHash?
|
||||
@NSManaged var magnetLink: SourceMagnetLink?
|
||||
@NSManaged var parentSource: Source?
|
||||
|
|
|
|||
|
|
@ -0,0 +1,13 @@
|
|||
//
|
||||
// SourceRequest+CoreDataClass.swift
|
||||
// Ferrite
|
||||
//
|
||||
// Created by Brian Dashore on 6/10/24.
|
||||
//
|
||||
//
|
||||
|
||||
import CoreData
|
||||
import Foundation
|
||||
|
||||
@objc(SourceRequest)
|
||||
public class SourceRequest: NSManagedObject {}
|
||||
|
|
@ -0,0 +1,25 @@
|
|||
//
|
||||
// SourceRequest+CoreDataProperties.swift
|
||||
// Ferrite
|
||||
//
|
||||
// Created by Brian Dashore on 6/10/24.
|
||||
//
|
||||
//
|
||||
|
||||
import CoreData
|
||||
import Foundation
|
||||
|
||||
public extension SourceRequest {
|
||||
@nonobjc class func fetchRequest() -> NSFetchRequest<SourceRequest> {
|
||||
NSFetchRequest<SourceRequest>(entityName: "SourceRequest")
|
||||
}
|
||||
|
||||
@NSManaged var method: String?
|
||||
@NSManaged var headers: [String: String]?
|
||||
@NSManaged var body: String?
|
||||
@NSManaged var parentHtmlParser: SourceHtmlParser?
|
||||
@NSManaged var parentRssParser: SourceRssParser?
|
||||
@NSManaged var parentJsonParser: SourceJsonParser?
|
||||
}
|
||||
|
||||
extension SourceRequest: Identifiable {}
|
||||
|
|
@ -17,6 +17,7 @@ public extension SourceRssParser {
|
|||
@NSManaged var items: String
|
||||
@NSManaged var rssUrl: String?
|
||||
@NSManaged var searchUrl: String
|
||||
@NSManaged var request: SourceRequest?
|
||||
@NSManaged var magnetHash: SourceMagnetHash?
|
||||
@NSManaged var magnetLink: SourceMagnetLink?
|
||||
@NSManaged var parentSource: Source?
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
|
||||
<model type="com.apple.IDECoreDataModeler.DataModel" documentVersion="1.0" lastSavedToolsVersion="21513" systemVersion="22D49" minimumToolsVersion="Automatic" sourceLanguage="Swift" userDefinedModelVersionIdentifier="">
|
||||
<model type="com.apple.IDECoreDataModeler.DataModel" documentVersion="1.0" lastSavedToolsVersion="22758" systemVersion="23F79" minimumToolsVersion="Automatic" sourceLanguage="Swift" userDefinedModelVersionIdentifier="">
|
||||
<entity name="Action" representedClassName="Action" syncable="YES">
|
||||
<attribute name="about" optional="YES" attributeType="String"/>
|
||||
<attribute name="author" attributeType="String" defaultValueString=""/>
|
||||
|
|
@ -106,6 +106,7 @@
|
|||
<relationship name="magnetHash" optional="YES" maxCount="1" deletionRule="Cascade" destinationEntity="SourceMagnetHash" inverseName="parentHtmlParser" inverseEntity="SourceMagnetHash"/>
|
||||
<relationship name="magnetLink" optional="YES" maxCount="1" deletionRule="Cascade" destinationEntity="SourceMagnetLink" inverseName="parentHtmlParser" inverseEntity="SourceMagnetLink"/>
|
||||
<relationship name="parentSource" optional="YES" maxCount="1" deletionRule="Nullify" destinationEntity="Source" inverseName="htmlParser" inverseEntity="Source"/>
|
||||
<relationship name="request" optional="YES" maxCount="1" deletionRule="Nullify" destinationEntity="SourceRequest" inverseName="parentHtmlParser" inverseEntity="SourceRequest"/>
|
||||
<relationship name="seedLeech" optional="YES" maxCount="1" deletionRule="Cascade" destinationEntity="SourceSeedLeech" inverseName="parentHtmlParser" inverseEntity="SourceSeedLeech"/>
|
||||
<relationship name="size" optional="YES" maxCount="1" deletionRule="Cascade" destinationEntity="SourceSize" inverseName="parentHtmlParser" inverseEntity="SourceSize"/>
|
||||
<relationship name="subName" optional="YES" maxCount="1" deletionRule="Nullify" destinationEntity="SourceSubName" inverseName="parentHtmlParser" inverseEntity="SourceSubName"/>
|
||||
|
|
@ -118,6 +119,7 @@
|
|||
<relationship name="magnetHash" optional="YES" maxCount="1" deletionRule="Cascade" destinationEntity="SourceMagnetHash" inverseName="parentJsonParser" inverseEntity="SourceMagnetHash"/>
|
||||
<relationship name="magnetLink" optional="YES" maxCount="1" deletionRule="Cascade" destinationEntity="SourceMagnetLink" inverseName="parentJsonParser" inverseEntity="SourceMagnetLink"/>
|
||||
<relationship name="parentSource" optional="YES" maxCount="1" deletionRule="Nullify" destinationEntity="Source" inverseName="jsonParser" inverseEntity="Source"/>
|
||||
<relationship name="request" optional="YES" maxCount="1" deletionRule="Nullify" destinationEntity="SourceRequest" inverseName="parentJsonParser" inverseEntity="SourceRequest"/>
|
||||
<relationship name="seedLeech" optional="YES" maxCount="1" deletionRule="Cascade" destinationEntity="SourceSeedLeech" inverseName="parentJsonParser" inverseEntity="SourceSeedLeech"/>
|
||||
<relationship name="size" optional="YES" maxCount="1" deletionRule="Cascade" destinationEntity="SourceSize" inverseName="parentJsonParser" inverseEntity="SourceSize"/>
|
||||
<relationship name="subName" optional="YES" maxCount="1" deletionRule="Nullify" destinationEntity="SourceSubName" inverseName="parentJsonParser" inverseEntity="SourceSubName"/>
|
||||
|
|
@ -134,6 +136,14 @@
|
|||
<relationship name="parentJsonParser" optional="YES" maxCount="1" deletionRule="Nullify" destinationEntity="SourceJsonParser" inverseName="magnetLink" inverseEntity="SourceJsonParser"/>
|
||||
<relationship name="parentRssParser" optional="YES" maxCount="1" deletionRule="Nullify" destinationEntity="SourceRssParser" inverseName="magnetLink" inverseEntity="SourceRssParser"/>
|
||||
</entity>
|
||||
<entity name="SourceRequest" representedClassName="SourceRequest" syncable="YES">
|
||||
<attribute name="body" optional="YES" attributeType="String" valueTransformerName="NSSecureUnarchiveFromData"/>
|
||||
<attribute name="headers" optional="YES" attributeType="Transformable" valueTransformerName="NSSecureUnarchiveFromData" customClassName="[String: String]"/>
|
||||
<attribute name="method" optional="YES" attributeType="String"/>
|
||||
<relationship name="parentHtmlParser" optional="YES" maxCount="1" deletionRule="Nullify" destinationEntity="SourceHtmlParser" inverseName="request" inverseEntity="SourceHtmlParser"/>
|
||||
<relationship name="parentJsonParser" optional="YES" maxCount="1" deletionRule="Nullify" destinationEntity="SourceJsonParser" inverseName="request" inverseEntity="SourceJsonParser"/>
|
||||
<relationship name="parentRssParser" optional="YES" maxCount="1" deletionRule="Nullify" destinationEntity="SourceRssParser" inverseName="request" inverseEntity="SourceRssParser"/>
|
||||
</entity>
|
||||
<entity name="SourceRssParser" representedClassName="SourceRssParser" syncable="YES">
|
||||
<attribute name="items" attributeType="String" defaultValueString=""/>
|
||||
<attribute name="rssUrl" optional="YES" attributeType="String"/>
|
||||
|
|
@ -141,6 +151,7 @@
|
|||
<relationship name="magnetHash" optional="YES" maxCount="1" deletionRule="Cascade" destinationEntity="SourceMagnetHash" inverseName="parentRssParser" inverseEntity="SourceMagnetHash"/>
|
||||
<relationship name="magnetLink" optional="YES" maxCount="1" deletionRule="Cascade" destinationEntity="SourceMagnetLink" inverseName="parentRssParser" inverseEntity="SourceMagnetLink"/>
|
||||
<relationship name="parentSource" optional="YES" maxCount="1" deletionRule="Nullify" destinationEntity="Source" inverseName="rssParser" inverseEntity="Source"/>
|
||||
<relationship name="request" optional="YES" maxCount="1" deletionRule="Nullify" destinationEntity="SourceRequest" inverseName="parentRssParser" inverseEntity="SourceRequest"/>
|
||||
<relationship name="seedLeech" optional="YES" maxCount="1" deletionRule="Cascade" destinationEntity="SourceSeedLeech" inverseName="parentRssParser" inverseEntity="SourceSeedLeech"/>
|
||||
<relationship name="size" optional="YES" maxCount="1" deletionRule="Cascade" destinationEntity="SourceSize" inverseName="parentRssParser" inverseEntity="SourceSize"/>
|
||||
<relationship name="subName" optional="YES" maxCount="1" deletionRule="Nullify" destinationEntity="SourceSubName" inverseName="parentRssParser" inverseEntity="SourceSubName"/>
|
||||
|
|
|
|||
|
|
@ -62,6 +62,7 @@ public struct SourceApiCredentialJson: Codable, Hashable, Sendable {
|
|||
|
||||
public struct SourceJsonParserJson: Codable, Hashable, Sendable {
|
||||
let searchUrl: String
|
||||
let request: SourceRequestJson?
|
||||
let results: String?
|
||||
let subResults: String?
|
||||
let title: SourceComplexQueryJson
|
||||
|
|
@ -75,6 +76,7 @@ public struct SourceJsonParserJson: Codable, Hashable, Sendable {
|
|||
public struct SourceRssParserJson: Codable, Hashable, Sendable {
|
||||
let rssUrl: String?
|
||||
let searchUrl: String
|
||||
let request: SourceRequestJson?
|
||||
let items: String
|
||||
let title: SourceComplexQueryJson
|
||||
let magnetHash: SourceComplexQueryJson?
|
||||
|
|
@ -86,6 +88,7 @@ public struct SourceRssParserJson: Codable, Hashable, Sendable {
|
|||
|
||||
public struct SourceHtmlParserJson: Codable, Hashable, Sendable {
|
||||
let searchUrl: String?
|
||||
let request: SourceRequestJson?
|
||||
let rows: String
|
||||
let title: SourceComplexQueryJson
|
||||
let magnet: SourceMagnetJson
|
||||
|
|
@ -117,3 +120,9 @@ public struct SourceSLJson: Codable, Hashable, Sendable {
|
|||
let seederRegex: String?
|
||||
let leecherRegex: String?
|
||||
}
|
||||
|
||||
public struct SourceRequestJson: Codable, Hashable, Sendable {
|
||||
let method: String?
|
||||
let headers: [String: String]?
|
||||
let body: String?
|
||||
}
|
||||
|
|
|
|||
|
|
@ -570,6 +570,7 @@ public class PluginManager: ObservableObject {
|
|||
newSource.api = newSourceApi
|
||||
}
|
||||
|
||||
// TODO: Migrate parser addition to a common protocol
|
||||
func addJsonParser(newSource: Source, jsonParserJson: SourceJsonParserJson) {
|
||||
let backgroundContext = PersistenceController.shared.backgroundContext
|
||||
|
||||
|
|
@ -578,6 +579,13 @@ public class PluginManager: ObservableObject {
|
|||
newSourceJsonParser.results = jsonParserJson.results
|
||||
newSourceJsonParser.subResults = jsonParserJson.subResults
|
||||
|
||||
if let requestJson = newSourceJsonParser.request {
|
||||
let newParserRequest = SourceRequest(context: backgroundContext)
|
||||
newParserRequest.method = requestJson.method
|
||||
newParserRequest.headers = requestJson.headers
|
||||
newParserRequest.body = requestJson.body
|
||||
}
|
||||
|
||||
// Tune these complex queries to the final JSON parser format
|
||||
if let magnetLinkJson = jsonParserJson.magnetLink {
|
||||
let newSourceMagnetLink = SourceMagnetLink(context: backgroundContext)
|
||||
|
|
@ -646,6 +654,13 @@ public class PluginManager: ObservableObject {
|
|||
newSourceRssParser.searchUrl = rssParserJson.searchUrl
|
||||
newSourceRssParser.items = rssParserJson.items
|
||||
|
||||
if let requestJson = newSourceRssParser.request {
|
||||
let newParserRequest = SourceRequest(context: backgroundContext)
|
||||
newParserRequest.method = requestJson.method
|
||||
newParserRequest.headers = requestJson.headers
|
||||
newParserRequest.body = requestJson.body
|
||||
}
|
||||
|
||||
if let magnetLinkJson = rssParserJson.magnetLink {
|
||||
let newSourceMagnetLink = SourceMagnetLink(context: backgroundContext)
|
||||
newSourceMagnetLink.query = magnetLinkJson.query
|
||||
|
|
@ -726,6 +741,16 @@ public class PluginManager: ObservableObject {
|
|||
newSourceHtmlParser.subName = newSourceSubName
|
||||
}
|
||||
|
||||
if let requestJson = htmlParserJson.request {
|
||||
print(requestJson)
|
||||
let newParserRequest = SourceRequest(context: backgroundContext)
|
||||
newParserRequest.method = requestJson.method
|
||||
newParserRequest.headers = requestJson.headers
|
||||
newParserRequest.body = requestJson.body
|
||||
|
||||
newSourceHtmlParser.request = newParserRequest
|
||||
}
|
||||
|
||||
// Adds a title complex query
|
||||
let newSourceTitle = SourceTitle(context: backgroundContext)
|
||||
newSourceTitle.query = htmlParserJson.title.query
|
||||
|
|
|
|||
|
|
@ -66,6 +66,28 @@ class ScrapingViewModel: ObservableObject {
|
|||
await logManager?.error(description, showToast: false)
|
||||
}
|
||||
|
||||
// Substitutes the given string with an arbitrary parameter dictionary
|
||||
func substituteParams(_ input: String, with params: [String: String]) -> String {
|
||||
let replaced = params.reduce(input) { result, param -> String in
|
||||
result.replacingOccurrences(of: "{\(param.key)}", with: param.value)
|
||||
}
|
||||
|
||||
return replaced
|
||||
}
|
||||
|
||||
// Cleans a SourceRequest's body and headers to be substituted
|
||||
func cleanRequest(request: SourceRequest, params: [String: String]) -> SourceRequest {
|
||||
if let body = request.body {
|
||||
request.body = substituteParams(body, with: params)
|
||||
}
|
||||
|
||||
if let headers = request.headers {
|
||||
request.headers = headers.mapValues { substituteParams($0, with: params) }
|
||||
}
|
||||
|
||||
return request
|
||||
}
|
||||
|
||||
public func scanSources(sources: [Source], searchText: String, debridManager: DebridManager) async {
|
||||
await logManager?.info("Started scanning sources for query \"\(searchText)\"")
|
||||
|
||||
|
|
@ -160,19 +182,25 @@ class ScrapingViewModel: ObservableObject {
|
|||
return nil
|
||||
}
|
||||
|
||||
// Initial params dict to reference
|
||||
// More params are added here as needed
|
||||
var params: [String: String] = [
|
||||
"query": encodedQuery
|
||||
]
|
||||
|
||||
switch preferredParser {
|
||||
case .scraping:
|
||||
if let htmlParser = source.htmlParser {
|
||||
let replacedSearchUrl = htmlParser.searchUrl.map {
|
||||
$0
|
||||
.replacingOccurrences(of: "{query}", with: encodedQuery)
|
||||
substituteParams($0, with: params)
|
||||
}
|
||||
|
||||
let data = await handleUrls(
|
||||
website: website,
|
||||
replacedSearchUrl: replacedSearchUrl,
|
||||
fallbackUrls: source.fallbackUrls,
|
||||
sourceName: source.name
|
||||
sourceName: source.name,
|
||||
requestParams: htmlParser.request.map { cleanRequest(request: $0, params: params) }
|
||||
)
|
||||
|
||||
if let data,
|
||||
|
|
@ -183,23 +211,25 @@ class ScrapingViewModel: ObservableObject {
|
|||
}
|
||||
case .rss:
|
||||
if let rssParser = source.rssParser {
|
||||
let replacedSearchUrl = rssParser.searchUrl
|
||||
.replacingOccurrences(of: "{secret}", with: source.api?.clientSecret?.value ?? "")
|
||||
.replacingOccurrences(of: "{query}", with: encodedQuery)
|
||||
params.updateValue(source.api?.clientSecret?.value ?? "", forKey: "secret")
|
||||
|
||||
let replacedSearchUrl = substituteParams(rssParser.searchUrl, with: params)
|
||||
|
||||
// Do not use fallback URLs if the base URL isn't used
|
||||
let data: Data?
|
||||
if let rssUrl = rssParser.rssUrl {
|
||||
data = await fetchWebsiteData(
|
||||
urlString: rssUrl + replacedSearchUrl,
|
||||
sourceName: source.name
|
||||
sourceName: source.name,
|
||||
requestParams: rssParser.request
|
||||
)
|
||||
} else {
|
||||
data = await handleUrls(
|
||||
website: website,
|
||||
replacedSearchUrl: replacedSearchUrl,
|
||||
fallbackUrls: source.fallbackUrls,
|
||||
sourceName: source.name
|
||||
sourceName: source.name,
|
||||
requestParams: rssParser.request
|
||||
)
|
||||
}
|
||||
|
||||
|
|
@ -211,8 +241,7 @@ class ScrapingViewModel: ObservableObject {
|
|||
}
|
||||
case .siteApi:
|
||||
if let jsonParser = source.jsonParser {
|
||||
var replacedSearchUrl = jsonParser.searchUrl
|
||||
.replacingOccurrences(of: "{query}", with: encodedQuery)
|
||||
var replacedSearchUrl = substituteParams(jsonParser.searchUrl, with: params)
|
||||
|
||||
// Handle anything API related including tokens, client IDs, and appending the API URL
|
||||
// The source API key is for APIs that require extra credentials or use a different URL
|
||||
|
|
@ -248,7 +277,8 @@ class ScrapingViewModel: ObservableObject {
|
|||
website: passedUrl,
|
||||
replacedSearchUrl: replacedSearchUrl,
|
||||
fallbackUrls: source.fallbackUrls,
|
||||
sourceName: source.name
|
||||
sourceName: source.name,
|
||||
requestParams: jsonParser.request
|
||||
)
|
||||
|
||||
if let data {
|
||||
|
|
@ -263,16 +293,16 @@ class ScrapingViewModel: ObservableObject {
|
|||
}
|
||||
|
||||
// Checks the base URL for any website data then iterates through the fallback URLs
|
||||
func handleUrls(website: String, replacedSearchUrl: String?, fallbackUrls: [String]?, sourceName: String) async -> Data? {
|
||||
func handleUrls(website: String, replacedSearchUrl: String?, fallbackUrls: [String]?, sourceName: String, requestParams: SourceRequest?) async -> Data? {
|
||||
let fetchUrl = website + (replacedSearchUrl.map { $0 } ?? "")
|
||||
if let data = await fetchWebsiteData(urlString: fetchUrl, sourceName: sourceName) {
|
||||
if let data = await fetchWebsiteData(urlString: fetchUrl, sourceName: sourceName, requestParams: requestParams) {
|
||||
return data
|
||||
}
|
||||
|
||||
if let fallbackUrls {
|
||||
for fallbackUrl in fallbackUrls {
|
||||
let fetchUrl = fallbackUrl + (replacedSearchUrl.map { $0 } ?? "")
|
||||
if let data = await fetchWebsiteData(urlString: fetchUrl, sourceName: sourceName) {
|
||||
if let data = await fetchWebsiteData(urlString: fetchUrl, sourceName: sourceName, requestParams: requestParams) {
|
||||
return data
|
||||
}
|
||||
}
|
||||
|
|
@ -298,8 +328,7 @@ class ScrapingViewModel: ObservableObject {
|
|||
|
||||
// Fetch a new credential if it's expired or doesn't exist yet
|
||||
if let value = credential.value, !isExpired {
|
||||
return searchUrl
|
||||
.replacingOccurrences(of: replacement, with: value)
|
||||
return substituteParams(searchUrl, with: [replacement: value])
|
||||
} else if
|
||||
credential.value == nil || isExpired,
|
||||
let credentialUrl = credential.urlString,
|
||||
|
|
@ -369,7 +398,7 @@ class ScrapingViewModel: ObservableObject {
|
|||
}
|
||||
|
||||
// Fetches the data for a URL
|
||||
public func fetchWebsiteData(urlString: String, sourceName: String) async -> Data? {
|
||||
public func fetchWebsiteData(urlString: String, sourceName: String, requestParams: SourceRequest?) async -> Data? {
|
||||
guard let url = URL(string: urlString.trimmingCharacters(in: .whitespacesAndNewlines)) else {
|
||||
await sendSourceError("\(sourceName): Source doesn't contain a valid URL, contact the source dev!")
|
||||
|
||||
|
|
@ -388,7 +417,12 @@ class ScrapingViewModel: ObservableObject {
|
|||
}
|
||||
}
|
||||
|
||||
let request = URLRequest(url: url, timeoutInterval: timeout)
|
||||
var request = URLRequest(url: url, timeoutInterval: timeout)
|
||||
request.httpMethod = requestParams?.method
|
||||
request.httpBody = requestParams?.body?.data(using: .utf8)
|
||||
requestParams?.headers?.forEach { field, value in
|
||||
request.addValue(value, forHTTPHeaderField: field)
|
||||
}
|
||||
|
||||
do {
|
||||
let (data, _) = try await URLSession.shared.data(for: request)
|
||||
|
|
@ -800,7 +834,7 @@ class ScrapingViewModel: ObservableObject {
|
|||
|
||||
let replacedMagnetUrl = externalMagnetUrl.starts(with: "/") ? website + externalMagnetUrl : externalMagnetUrl
|
||||
guard
|
||||
let data = await fetchWebsiteData(urlString: replacedMagnetUrl, sourceName: source.name),
|
||||
let data = await fetchWebsiteData(urlString: replacedMagnetUrl, sourceName: source.name, requestParams: htmlParser.request),
|
||||
let magnetHtml = String(data: data, encoding: .utf8)
|
||||
else {
|
||||
continue
|
||||
|
|
|
|||
Loading…
Reference in a new issue