diff --git a/Ferrite/DataManagement/Classes/Source+CoreDataProperties.swift b/Ferrite/DataManagement/Classes/Source+CoreDataProperties.swift
index 4a1a0cf..97b85ee 100644
--- a/Ferrite/DataManagement/Classes/Source+CoreDataProperties.swift
+++ b/Ferrite/DataManagement/Classes/Source+CoreDataProperties.swift
@@ -16,6 +16,7 @@ public extension Source {
@NSManaged var id: UUID
@NSManaged var baseUrl: String?
+ @NSManaged var fallbackUrls: [String]?
@NSManaged var dynamicBaseUrl: Bool
@NSManaged var enabled: Bool
@NSManaged var name: String
diff --git a/Ferrite/DataManagement/FerriteDB.xcdatamodeld/FerriteDB.xcdatamodel/contents b/Ferrite/DataManagement/FerriteDB.xcdatamodeld/FerriteDB.xcdatamodel/contents
index c2b3812..99cfeeb 100644
--- a/Ferrite/DataManagement/FerriteDB.xcdatamodeld/FerriteDB.xcdatamodel/contents
+++ b/Ferrite/DataManagement/FerriteDB.xcdatamodeld/FerriteDB.xcdatamodel/contents
@@ -5,6 +5,7 @@
+
diff --git a/Ferrite/Models/SourceModels.swift b/Ferrite/Models/SourceModels.swift
index daa30ca..dd28415 100644
--- a/Ferrite/Models/SourceModels.swift
+++ b/Ferrite/Models/SourceModels.swift
@@ -22,6 +22,7 @@ public struct SourceJson: Codable, Hashable {
let name: String
let version: Int16
let baseUrl: String?
+ let fallbackUrls: [String]?
var dynamicBaseUrl: Bool?
var author: String?
var listId: UUID?
diff --git a/Ferrite/ViewModels/ScrapingViewModel.swift b/Ferrite/ViewModels/ScrapingViewModel.swift
index 4beec42..df646da 100644
--- a/Ferrite/ViewModels/ScrapingViewModel.swift
+++ b/Ferrite/ViewModels/ScrapingViewModel.swift
@@ -73,10 +73,16 @@ class ScrapingViewModel: ObservableObject {
switch preferredParser {
case .scraping:
if let htmlParser = source.htmlParser {
- let urlString = baseUrl + htmlParser.searchUrl
+ let replacedSearchUrl = htmlParser.searchUrl
.replacingOccurrences(of: "{query}", with: encodedQuery)
- if let data = await fetchWebsiteData(urlString: urlString),
+ let data = await handleUrls(
+ baseUrl: baseUrl,
+ replacedSearchUrl: replacedSearchUrl,
+ fallbackUrls: source.fallbackUrls
+ )
+
+ if let data = data,
let html = String(data: data, encoding: .utf8)
{
let sourceResults = await scrapeHtml(source: source, baseUrl: baseUrl, html: html)
@@ -89,10 +95,19 @@ class ScrapingViewModel: ObservableObject {
.replacingOccurrences(of: "{secret}", with: source.api?.clientSecret?.value ?? "")
.replacingOccurrences(of: "{query}", with: encodedQuery)
- // If there is an RSS base URL, use that instead
- let urlString = (rssParser.rssUrl ?? baseUrl) + replacedSearchUrl
+ // Do not use fallback URLs if the base URL isn't used
+ let data: Data?
+ if let rssUrl = rssParser.rssUrl {
+ data = await fetchWebsiteData(urlString: rssUrl + replacedSearchUrl)
+ } else {
+ data = await handleUrls(
+ baseUrl: baseUrl,
+ replacedSearchUrl: replacedSearchUrl,
+ fallbackUrls: source.fallbackUrls
+ )
+ }
- if let data = await fetchWebsiteData(urlString: urlString),
+ if let data = data,
let rss = String(data: data, encoding: .utf8)
{
let sourceResults = scrapeRss(source: source, rss: rss)
@@ -131,9 +146,14 @@ class ScrapingViewModel: ObservableObject {
}
}
- let urlString = (source.api?.apiUrl ?? baseUrl) + replacedSearchUrl
+ let passedUrl = source.api?.apiUrl ?? baseUrl
+ let data = await handleUrls(
+ baseUrl: passedUrl,
+ replacedSearchUrl: replacedSearchUrl,
+ fallbackUrls: source.fallbackUrls
+ )
- if let data = await fetchWebsiteData(urlString: urlString) {
+ if let data = data {
let sourceResults = scrapeJson(source: source, jsonData: data)
tempResults += sourceResults
}
@@ -152,6 +172,23 @@ class ScrapingViewModel: ObservableObject {
searchResults = tempResults
}
+ // Checks the base URL for any website data then iterates through the fallback URLs
+ func handleUrls(baseUrl: String, replacedSearchUrl: String, fallbackUrls: [String]?) async -> Data? {
+ if let data = await fetchWebsiteData(urlString: baseUrl + replacedSearchUrl) {
+ return data
+ }
+
+ if let fallbackUrls = fallbackUrls {
+ for fallbackUrl in fallbackUrls {
+ if let data = await fetchWebsiteData(urlString: fallbackUrl + replacedSearchUrl) {
+ return data
+ }
+ }
+ }
+
+ return nil
+ }
+
public func handleApiCredential(_ credential: SourceApiCredential,
replacement: String,
searchUrl: String,
@@ -220,8 +257,18 @@ class ScrapingViewModel: ObservableObject {
return String(data: data, encoding: .utf8)
}
} catch {
+ let error = error as NSError
+
Task { @MainActor in
- toastModel?.toastDescription = "Error in fetching an API credential \(error)"
+ switch error.code {
+ case -999:
+ toastModel?.toastType = .info
+ toastModel?.toastDescription = "Search cancelled"
+ case -1001:
+ toastModel?.toastDescription = "Credentials request timed out"
+ default:
+ toastModel?.toastDescription = "Error in fetching an API credential \(error)"
+ }
}
print("Error in fetching an API credential \(error)")
@@ -241,8 +288,10 @@ class ScrapingViewModel: ObservableObject {
return nil
}
+ let request = URLRequest(url: url, timeoutInterval: 15)
+
do {
- let (data, _) = try await URLSession.shared.data(from: url)
+ let (data, _) = try await URLSession.shared.data(for: request)
return data
} catch {
let error = error as NSError
@@ -252,8 +301,10 @@ class ScrapingViewModel: ObservableObject {
case -999:
toastModel?.toastType = .info
toastModel?.toastDescription = "Search cancelled"
+ case -1001:
+ toastModel?.toastDescription = "Data request timed out. Trying fallback URLs if present."
default:
- toastModel?.toastDescription = "Error in fetching data \(error)"
+ toastModel?.toastDescription = "Error in fetching website data \(error)"
}
}
print("Error in fetching data \(error)")
@@ -833,21 +884,49 @@ class ScrapingViewModel: ObservableObject {
func cleanApiCreds(api: SourceApi) {
let backgroundContext = PersistenceController.shared.backgroundContext
- var clientIdReset = false
- var clientSecretReset = false
+ let hasCredentials = api.clientId != nil || api.clientSecret != nil
+ let clientIdReset: Bool
+ let clientSecretReset: Bool
+
+ var responseArray = ["Could not fetch API results"]
if let clientId = api.clientId, !clientId.dynamic {
clientId.value = nil
clientIdReset = true
+ } else {
+ clientIdReset = false
}
if let clientSecret = api.clientSecret, !clientSecret.dynamic {
clientSecret.value = nil
clientSecretReset = true
+ } else {
+ clientSecretReset = false
}
- toastModel?.toastDescription =
- "Could not fetch results, your \(clientIdReset ? "client ID" : "") \(clientIdReset && clientSecretReset ? "and" : "") \(clientSecretReset ? "token" : "") was automatically reset. Make sure all credentials are correct in the source's settings!"
+ if hasCredentials {
+ responseArray.append("your")
+
+ if clientIdReset {
+ responseArray.append("client ID")
+ }
+
+ if clientIdReset && clientSecretReset {
+ responseArray.append("and")
+ }
+
+ if clientSecretReset {
+ responseArray.append("token")
+ }
+
+ responseArray.append("was automatically reset.")
+
+ if !(clientIdReset || clientSecretReset) {
+ responseArray.append("Make sure all credentials are correct in the source's settings!")
+ }
+ }
+
+ toastModel?.toastDescription = responseArray.joined()
PersistenceController.shared.save(backgroundContext)
}
diff --git a/Ferrite/ViewModels/SourceManager.swift b/Ferrite/ViewModels/SourceManager.swift
index 07c44fb..da6ea1a 100644
--- a/Ferrite/ViewModels/SourceManager.swift
+++ b/Ferrite/ViewModels/SourceManager.swift
@@ -96,6 +96,7 @@ public class SourceManager: ObservableObject {
newSource.version = sourceJson.version
newSource.dynamicBaseUrl = dynamicBaseUrl
newSource.baseUrl = sourceJson.baseUrl
+ newSource.fallbackUrls = dynamicBaseUrl ? nil : sourceJson.fallbackUrls
newSource.author = sourceJson.author ?? "Unknown"
newSource.listId = sourceJson.listId
newSource.trackers = sourceJson.trackers