Creepso/Ferrite-backup
1
0
Fork 0
Code Issues 5 Pull requests Projects Releases 15 Packages Wiki Activity Actions

Sources: Add fallback URLs

Browse source 
If a website times out, use the fallback options. The data URL
request now has a hard timeout of 15 seconds.

This only works for the base URL of a source, if an RSS url or
API url is provided, fallback URLs won't be used and the request
will fail.

Signed-off-by: kingbri <bdashore3@gmail.com>
This commit is contained in:
kingbri 2022-08-24 11:36:32 -04:00 committed by kingbri
parent e0182a700f
commit 79d88ffab6
5 changed files with 97 additions and 14 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
Ferrite/DataManagement/Classes/Source+CoreDataProperties.swift
View file

@ -16,6 +16,7 @@ public extension Source {
@NSManaged var id: UUID
@NSManaged var baseUrl: String?
@NSManaged var fallbackUrls: [String]?
@NSManaged var dynamicBaseUrl: Bool
@NSManaged var enabled: Bool
@NSManaged var name: String

1
Ferrite/DataManagement/FerriteDB.xcdatamodeld/FerriteDB.xcdatamodel/contents
View file

@ -5,6 +5,7 @@
<attribute name="baseUrl" optional="YES" attributeType="String"/>
<attribute name="dynamicBaseUrl" optional="YES" attributeType="Boolean" usesScalarValueType="YES"/>
<attribute name="enabled" attributeType="Boolean" defaultValueString="NO" usesScalarValueType="YES"/>
<attribute name="fallbackUrls" optional="YES" attributeType="Transformable" valueTransformerName="NSSecureUnarchiveFromData" customClassName="[String]"/>
<attribute name="id" attributeType="UUID" usesScalarValueType="NO"/>
<attribute name="listId" optional="YES" attributeType="UUID" usesScalarValueType="NO"/>
<attribute name="name" attributeType="String" defaultValueString=""/>

1
Ferrite/Models/SourceModels.swift
View file

@ -22,6 +22,7 @@ public struct SourceJson: Codable, Hashable {
let name: String
let version: Int16
let baseUrl: String?
let fallbackUrls: [String]?
var dynamicBaseUrl: Bool?
var author: String?
var listId: UUID?

107
Ferrite/ViewModels/ScrapingViewModel.swift
View file

@ -73,10 +73,16 @@ class ScrapingViewModel: ObservableObject {
switch preferredParser {
case .scraping:
if let htmlParser = source.htmlParser {
let urlString = baseUrl + htmlParser.searchUrl
let replacedSearchUrl = htmlParser.searchUrl
.replacingOccurrences(of: "{query}", with: encodedQuery)
if let data = await fetchWebsiteData(urlString: urlString),
let data = await handleUrls(
baseUrl: baseUrl,
replacedSearchUrl: replacedSearchUrl,
fallbackUrls: source.fallbackUrls
)
if let data = data,
let html = String(data: data, encoding: .utf8)
{
let sourceResults = await scrapeHtml(source: source, baseUrl: baseUrl, html: html)
@ -89,10 +95,19 @@ class ScrapingViewModel: ObservableObject {
.replacingOccurrences(of: "{secret}", with: source.api?.clientSecret?.value ?? "")
.replacingOccurrences(of: "{query}", with: encodedQuery)
// If there is an RSS base URL, use that instead
let urlString = (rssParser.rssUrl ?? baseUrl) + replacedSearchUrl
// Do not use fallback URLs if the base URL isn't used
let data: Data?
if let rssUrl = rssParser.rssUrl {
data = await fetchWebsiteData(urlString: rssUrl + replacedSearchUrl)
} else {
data = await handleUrls(
baseUrl: baseUrl,
replacedSearchUrl: replacedSearchUrl,
fallbackUrls: source.fallbackUrls
)
}
if let data = await fetchWebsiteData(urlString: urlString),
if let data = data,
let rss = String(data: data, encoding: .utf8)
{
let sourceResults = scrapeRss(source: source, rss: rss)
@ -131,9 +146,14 @@ class ScrapingViewModel: ObservableObject {
}
}
let urlString = (source.api?.apiUrl ?? baseUrl) + replacedSearchUrl
let passedUrl = source.api?.apiUrl ?? baseUrl
let data = await handleUrls(
baseUrl: passedUrl,
replacedSearchUrl: replacedSearchUrl,
fallbackUrls: source.fallbackUrls
)
if let data = await fetchWebsiteData(urlString: urlString) {
if let data = data {
let sourceResults = scrapeJson(source: source, jsonData: data)
tempResults += sourceResults
}
@ -152,6 +172,23 @@ class ScrapingViewModel: ObservableObject {
searchResults = tempResults
}
// Checks the base URL for any website data then iterates through the fallback URLs
func handleUrls(baseUrl: String, replacedSearchUrl: String, fallbackUrls: [String]?) async -> Data? {
if let data = await fetchWebsiteData(urlString: baseUrl + replacedSearchUrl) {
return data
}
if let fallbackUrls = fallbackUrls {
for fallbackUrl in fallbackUrls {
if let data = await fetchWebsiteData(urlString: fallbackUrl + replacedSearchUrl) {
return data
}
}
}
return nil
}
public func handleApiCredential(_ credential: SourceApiCredential,
replacement: String,
searchUrl: String,
@ -220,8 +257,18 @@ class ScrapingViewModel: ObservableObject {
return String(data: data, encoding: .utf8)
}
} catch {
let error = error as NSError
Task { @MainActor in
toastModel?.toastDescription = "Error in fetching an API credential \(error)"
switch error.code {
case -999:
toastModel?.toastType = .info
toastModel?.toastDescription = "Search cancelled"
case -1001:
toastModel?.toastDescription = "Credentials request timed out"
default:
toastModel?.toastDescription = "Error in fetching an API credential \(error)"
}
}
print("Error in fetching an API credential \(error)")
@ -241,8 +288,10 @@ class ScrapingViewModel: ObservableObject {
return nil
}
let request = URLRequest(url: url, timeoutInterval: 15)
do {
let (data, _) = try await URLSession.shared.data(from: url)
let (data, _) = try await URLSession.shared.data(for: request)
return data
} catch {
let error = error as NSError
@ -252,8 +301,10 @@ class ScrapingViewModel: ObservableObject {
case -999:
toastModel?.toastType = .info
toastModel?.toastDescription = "Search cancelled"
case -1001:
toastModel?.toastDescription = "Data request timed out. Trying fallback URLs if present."
default:
toastModel?.toastDescription = "Error in fetching data \(error)"
toastModel?.toastDescription = "Error in fetching website data \(error)"
}
}
print("Error in fetching data \(error)")
@ -833,21 +884,49 @@ class ScrapingViewModel: ObservableObject {
func cleanApiCreds(api: SourceApi) {
let backgroundContext = PersistenceController.shared.backgroundContext
var clientIdReset = false
var clientSecretReset = false
let hasCredentials = api.clientId != nil || api.clientSecret != nil
let clientIdReset: Bool
let clientSecretReset: Bool
var responseArray = ["Could not fetch API results"]
if let clientId = api.clientId, !clientId.dynamic {
clientId.value = nil
clientIdReset = true
} else {
clientIdReset = false
}
if let clientSecret = api.clientSecret, !clientSecret.dynamic {
clientSecret.value = nil
clientSecretReset = true
} else {
clientSecretReset = false
}
toastModel?.toastDescription =
"Could not fetch results, your \(clientIdReset ? "client ID" : "") \(clientIdReset && clientSecretReset ? "and" : "") \(clientSecretReset ? "token" : "") was automatically reset. Make sure all credentials are correct in the source's settings!"
if hasCredentials {
responseArray.append("your")
if clientIdReset {
responseArray.append("client ID")
}
if clientIdReset && clientSecretReset {
responseArray.append("and")
}
if clientSecretReset {
responseArray.append("token")
}
responseArray.append("was automatically reset.")
if !(clientIdReset || clientSecretReset) {
responseArray.append("Make sure all credentials are correct in the source's settings!")
}
}
toastModel?.toastDescription = responseArray.joined()
PersistenceController.shared.save(backgroundContext)
}

1
Ferrite/ViewModels/SourceManager.swift
View file

@ -96,6 +96,7 @@ public class SourceManager: ObservableObject {
newSource.version = sourceJson.version
newSource.dynamicBaseUrl = dynamicBaseUrl
newSource.baseUrl = sourceJson.baseUrl
newSource.fallbackUrls = dynamicBaseUrl ? nil : sourceJson.fallbackUrls
newSource.author = sourceJson.author ?? "Unknown"
newSource.listId = sourceJson.listId
newSource.trackers = sourceJson.trackers