Creepso/Ferrite-backup
1
0
Fork 0
Code Issues 5 Pull requests Projects Releases 15 Packages Wiki Activity Actions

Sources: Add subName and fixup

Browse source 
The subName parameter is for aggregate sources that pull from a
child website. Make it so it's possible to include that child
site in parsers.

Also remove the magnet link/hash requirement since it's filtered out
anyways after results are fetched.

Signed-off-by: kingbri <bdashore3@proton.me>
This commit is contained in:
kingbri 2023-02-08 22:10:02 -05:00
parent 41572362c7
commit 88a2dc9742
7 changed files with 93 additions and 28 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
Ferrite/DataManagement/Classes/SourceHtmlParser+CoreDataProperties.swift
View file

@ -22,6 +22,7 @@ public extension SourceHtmlParser {
@NSManaged var seedLeech: SourceSeedLeech?
@NSManaged var size: SourceSize?
@NSManaged var title: SourceTitle?
@NSManaged var subName: SourceSubName?
}
extension SourceHtmlParser: Identifiable {}

1
Ferrite/DataManagement/Classes/SourceJsonParser+CoreDataProperties.swift
View file

@ -23,6 +23,7 @@ public extension SourceJsonParser {
@NSManaged var seedLeech: SourceSeedLeech?
@NSManaged var size: SourceSize?
@NSManaged var title: SourceTitle?
@NSManaged var subName: SourceSubName?
}
extension SourceJsonParser: Identifiable {}

1
Ferrite/DataManagement/Classes/SourceRssParser+CoreDataProperties.swift
View file

@ -23,6 +23,7 @@ public extension SourceRssParser {
@NSManaged var seedLeech: SourceSeedLeech?
@NSManaged var size: SourceSize?
@NSManaged var title: SourceTitle?
@NSManaged var subName: SourceSubName?
}
extension SourceRssParser: Identifiable {}

8
Ferrite/DataManagement/FerriteDB.xcdatamodeld/FerriteDB_v2.xcdatamodel/contents
View file

@ -99,6 +99,7 @@
<relationship name="parentSource" optional="YES" maxCount="1" deletionRule="Nullify" destinationEntity="Source" inverseName="htmlParser" inverseEntity="Source"/>
<relationship name="seedLeech" optional="YES" maxCount="1" deletionRule="Cascade" destinationEntity="SourceSeedLeech" inverseName="parentHtmlParser" inverseEntity="SourceSeedLeech"/>
<relationship name="size" optional="YES" maxCount="1" deletionRule="Cascade" destinationEntity="SourceSize" inverseName="parentHtmlParser" inverseEntity="SourceSize"/>
<relationship name="subName" optional="YES" maxCount="1" deletionRule="Nullify" destinationEntity="SourceSubName" inverseName="parentHtmlParser" inverseEntity="SourceSubName"/>
<relationship name="title" optional="YES" maxCount="1" deletionRule="Cascade" destinationEntity="SourceTitle" inverseName="parentHtmlParser" inverseEntity="SourceTitle"/>
</entity>
<entity name="SourceJsonParser" representedClassName="SourceJsonParser" syncable="YES">
@ -110,6 +111,7 @@
<relationship name="parentSource" optional="YES" maxCount="1" deletionRule="Nullify" destinationEntity="Source" inverseName="jsonParser" inverseEntity="Source"/>
<relationship name="seedLeech" optional="YES" maxCount="1" deletionRule="Cascade" destinationEntity="SourceSeedLeech" inverseName="parentJsonParser" inverseEntity="SourceSeedLeech"/>
<relationship name="size" optional="YES" maxCount="1" deletionRule="Cascade" destinationEntity="SourceSize" inverseName="parentJsonParser" inverseEntity="SourceSize"/>
<relationship name="subName" optional="YES" maxCount="1" deletionRule="Nullify" destinationEntity="SourceSubName" inverseName="parentJsonParser" inverseEntity="SourceSubName"/>
<relationship name="title" optional="YES" maxCount="1" deletionRule="Cascade" destinationEntity="SourceTitle" inverseName="parentJsonParser" inverseEntity="SourceTitle"/>
</entity>
<entity name="SourceMagnetHash" representedClassName="SourceMagnetHash" parentEntity="SourceComplexQuery" syncable="YES" codeGenerationType="class">
@ -132,6 +134,7 @@
<relationship name="parentSource" optional="YES" maxCount="1" deletionRule="Nullify" destinationEntity="Source" inverseName="rssParser" inverseEntity="Source"/>
<relationship name="seedLeech" optional="YES" maxCount="1" deletionRule="Cascade" destinationEntity="SourceSeedLeech" inverseName="parentRssParser" inverseEntity="SourceSeedLeech"/>
<relationship name="size" optional="YES" maxCount="1" deletionRule="Cascade" destinationEntity="SourceSize" inverseName="parentRssParser" inverseEntity="SourceSize"/>
<relationship name="subName" optional="YES" maxCount="1" deletionRule="Nullify" destinationEntity="SourceSubName" inverseName="parentRssParser" inverseEntity="SourceSubName"/>
<relationship name="title" optional="YES" maxCount="1" deletionRule="Cascade" destinationEntity="SourceTitle" inverseName="parentRssParser" inverseEntity="SourceTitle"/>
</entity>
<entity name="SourceSeedLeech" representedClassName="SourceSeedLeech" syncable="YES">
@ -151,6 +154,11 @@
<relationship name="parentJsonParser" optional="YES" maxCount="1" deletionRule="Nullify" destinationEntity="SourceJsonParser" inverseName="size" inverseEntity="SourceJsonParser"/>
<relationship name="parentRssParser" optional="YES" maxCount="1" deletionRule="Nullify" destinationEntity="SourceRssParser" inverseName="size" inverseEntity="SourceRssParser"/>
</entity>
<entity name="SourceSubName" representedClassName="SourceSubName" parentEntity="SourceComplexQuery" syncable="YES" codeGenerationType="class">
<relationship name="parentHtmlParser" optional="YES" maxCount="1" deletionRule="Nullify" destinationEntity="SourceHtmlParser" inverseName="subName" inverseEntity="SourceHtmlParser"/>
<relationship name="parentJsonParser" optional="YES" maxCount="1" deletionRule="Nullify" destinationEntity="SourceJsonParser" inverseName="subName" inverseEntity="SourceJsonParser"/>
<relationship name="parentRssParser" optional="YES" maxCount="1" deletionRule="Nullify" destinationEntity="SourceRssParser" inverseName="subName" inverseEntity="SourceRssParser"/>
</entity>
<entity name="SourceTitle" representedClassName="SourceTitle" parentEntity="SourceComplexQuery" syncable="YES" codeGenerationType="class">
<relationship name="parentHtmlParser" optional="YES" maxCount="1" deletionRule="Nullify" destinationEntity="SourceHtmlParser" inverseName="title" inverseEntity="SourceHtmlParser"/>
<relationship name="parentJsonParser" optional="YES" maxCount="1" deletionRule="Nullify" destinationEntity="SourceJsonParser" inverseName="title" inverseEntity="SourceJsonParser"/>

27
Ferrite/Models/SourceModels.swift
View file

@ -18,7 +18,7 @@ public struct SourceJson: Codable, Hashable, Sendable, PluginJson {
let minVersion: String?
let baseUrl: String?
let fallbackUrls: [String]?
var dynamicBaseUrl: Bool?
let dynamicBaseUrl: Bool?
let trackers: [String]?
let api: SourceApiJson?
let jsonParser: SourceJsonParserJson?
@ -62,10 +62,11 @@ public struct SourceJsonParserJson: Codable, Hashable, Sendable {
let searchUrl: String
let results: String?
let subResults: String?
let magnetHash: SouceComplexQueryJson?
let magnetLink: SouceComplexQueryJson?
let title: SouceComplexQueryJson?
let size: SouceComplexQueryJson?
let magnetHash: SourceComplexQueryJson?
let magnetLink: SourceComplexQueryJson?
let subName: SourceComplexQueryJson?
let title: SourceComplexQueryJson?
let size: SourceComplexQueryJson?
let sl: SourceSLJson?
}
@ -73,10 +74,11 @@ public struct SourceRssParserJson: Codable, Hashable, Sendable {
let rssUrl: String?
let searchUrl: String
let items: String
let magnetHash: SouceComplexQueryJson?
let magnetLink: SouceComplexQueryJson?
let title: SouceComplexQueryJson?
let size: SouceComplexQueryJson?
let magnetHash: SourceComplexQueryJson?
let magnetLink: SourceComplexQueryJson?
let subName: SourceComplexQueryJson?
let title: SourceComplexQueryJson?
let size: SourceComplexQueryJson?
let sl: SourceSLJson?
}
@ -84,12 +86,13 @@ public struct SourceHtmlParserJson: Codable, Hashable, Sendable {
let searchUrl: String
let rows: String
let magnet: SourceMagnetJson
let title: SouceComplexQueryJson?
let size: SouceComplexQueryJson?
let subName: SourceComplexQueryJson?
let title: SourceComplexQueryJson?
let size: SourceComplexQueryJson?
let sl: SourceSLJson?
}
public struct SouceComplexQueryJson: Codable, Hashable, Sendable {
public struct SourceComplexQueryJson: Codable, Hashable, Sendable {
let query: String
let discriminator: String?
let attribute: String?

34
Ferrite/ViewModels/PluginManager.swift
View file

@ -45,6 +45,7 @@ public class PluginManager: ObservableObject {
minVersion: inputJson.minVersion,
baseUrl: inputJson.baseUrl,
fallbackUrls: inputJson.fallbackUrls,
dynamicBaseUrl: inputJson.dynamicBaseUrl,
trackers: inputJson.trackers,
api: inputJson.api,
jsonParser: inputJson.jsonParser,
@ -257,7 +258,6 @@ public class PluginManager: ObservableObject {
// If there's no base URL and it isn't dynamic, return before any transactions occur
let dynamicBaseUrl = sourceJson.dynamicBaseUrl ?? false
if !dynamicBaseUrl, sourceJson.baseUrl == nil {
await toastModel?.updateToastDescription("Not adding this source because base URL parameters are malformed. Please contact the source dev.")
print("Not adding source \(sourceJson.name) because base URL parameters are malformed")
@ -401,6 +401,15 @@ public class PluginManager: ObservableObject {
newSourceJsonParser.magnetHash = newSourceMagnetHash
}
if let subNameJson = jsonParserJson.subName {
let newSourceSubName = SourceSubName(context: backgroundContext)
newSourceSubName.query = subNameJson.query
newSourceSubName.attribute = subNameJson.query
newSourceSubName.discriminator = subNameJson.discriminator
newSourceJsonParser.subName = newSourceSubName
}
if let titleJson = jsonParserJson.title {
let newSourceTitle = SourceTitle(context: backgroundContext)
newSourceTitle.query = titleJson.query
@ -448,6 +457,7 @@ public class PluginManager: ObservableObject {
newSourceMagnetLink.query = magnetLinkJson.query
newSourceMagnetLink.attribute = magnetLinkJson.attribute ?? "text"
newSourceMagnetLink.discriminator = magnetLinkJson.discriminator
newSourceMagnetLink.regex = magnetLinkJson.regex
newSourceRssParser.magnetLink = newSourceMagnetLink
}
@ -457,15 +467,27 @@ public class PluginManager: ObservableObject {
newSourceMagnetHash.query = magnetHashJson.query
newSourceMagnetHash.attribute = magnetHashJson.attribute ?? "text"
newSourceMagnetHash.discriminator = magnetHashJson.discriminator
newSourceMagnetHash.regex = magnetHashJson.regex
newSourceRssParser.magnetHash = newSourceMagnetHash
}
if let subNameJson = rssParserJson.subName {
let newSourceSubName = SourceSubName(context: backgroundContext)
newSourceSubName.query = subNameJson.query
newSourceSubName.attribute = subNameJson.attribute ?? "text"
newSourceSubName.discriminator = subNameJson.discriminator
newSourceSubName.regex = subNameJson.regex
newSourceRssParser.subName = newSourceSubName
}
if let titleJson = rssParserJson.title {
let newSourceTitle = SourceTitle(context: backgroundContext)
newSourceTitle.query = titleJson.query
newSourceTitle.attribute = titleJson.attribute ?? "text"
newSourceTitle.discriminator = titleJson.discriminator
newSourceTitle.regex = titleJson.regex
newSourceRssParser.title = newSourceTitle
}
@ -475,6 +497,7 @@ public class PluginManager: ObservableObject {
newSourceSize.query = sizeJson.query
newSourceSize.attribute = sizeJson.attribute ?? "text"
newSourceSize.discriminator = sizeJson.discriminator
newSourceSize.regex = sizeJson.regex
newSourceRssParser.size = newSourceSize
}
@ -502,6 +525,15 @@ public class PluginManager: ObservableObject {
newSourceHtmlParser.searchUrl = htmlParserJson.searchUrl
newSourceHtmlParser.rows = htmlParserJson.rows
if let subNameJson = htmlParserJson.subName {
let newSourceSubName = SourceSubName(context: backgroundContext)
newSourceSubName.query = subNameJson.query
newSourceSubName.attribute = subNameJson.attribute ?? "text"
newSourceSubName.regex = subNameJson.regex
newSourceHtmlParser.subName = newSourceSubName
}
// Adds a title complex query if present
if let titleJson = htmlParserJson.title {
let newSourceTitle = SourceTitle(context: backgroundContext)

49
Ferrite/ViewModels/ScrapingViewModel.swift
View file

@ -399,6 +399,12 @@ class ScrapingViewModel: ObservableObject {
}
}
var subName: String?
if let subNameParser = jsonParser.subName {
let rawSubName = result[subNameParser.query.components(separatedBy: ".")].rawValue
subName = rawSubName is NSNull ? nil : String(describing: rawSubName)
}
var link: String? = existingSearchResult?.magnet.link
if let magnetLinkParser = jsonParser.magnetLink, link == nil {
let rawLink = result[magnetLinkParser.query.components(separatedBy: ".")].rawValue
@ -432,7 +438,7 @@ class ScrapingViewModel: ObservableObject {
let result = SearchResult(
title: title,
source: source.name,
source: subName.map { "\(source.name) - \($0)" } ?? source.name,
size: size,
magnet: Magnet(hash: magnetHash, link: link, title: title, trackers: source.trackers),
seeders: seeders,
@ -474,6 +480,18 @@ class ScrapingViewModel: ObservableObject {
)
}
// Fetches the subName for the source if there is one
var subName: String?
if let subNameParser = rssParser.subName {
subName = try? runRssComplexQuery(
item: item,
query: subNameParser.query,
attribute: subNameParser.attribute,
discriminator: subNameParser.discriminator,
regexString: subNameParser.regex
)
}
var title: String?
if let titleParser = rssParser.title {
title = try? runRssComplexQuery(
@ -485,9 +503,9 @@ class ScrapingViewModel: ObservableObject {
)
}
var link: String?
var href: String?
if let magnetLinkParser = rssParser.magnetLink {
link = try? runRssComplexQuery(
href = try? runRssComplexQuery(
item: item,
query: magnetLinkParser.query,
attribute: magnetLinkParser.attribute,
@ -498,10 +516,6 @@ class ScrapingViewModel: ObservableObject {
continue
}
guard let href = link, href.starts(with: "magnet:") else {
continue
}
var size: String?
if let sizeParser = rssParser.size {
size = try? runRssComplexQuery(
@ -543,7 +557,7 @@ class ScrapingViewModel: ObservableObject {
let result = SearchResult(
title: title ?? "No title",
source: source.name,
source: subName.map { "\(source.name) - \($0)" } ?? source.name,
size: size ?? "",
magnet: Magnet(hash: magnetHash, link: href, title: title, trackers: source.trackers),
seeders: seeders,
@ -649,10 +663,6 @@ class ScrapingViewModel: ObservableObject {
href = link
}
if !href.starts(with: "magnet:") {
continue
}
// Fetches the episode/movie title
var title: String?
if let titleParser = htmlParser.title {
@ -664,8 +674,17 @@ class ScrapingViewModel: ObservableObject {
)
}
// Fetches the torrent's size
// TODO: Add int translation
var subName: String?
if let subNameParser = htmlParser.subName {
subName = try? runHtmlComplexQuery(
row: row,
query: subNameParser.query,
attribute: subNameParser.attribute,
regexString: subNameParser.regex
)
}
// Fetches the size
var size: String?
if let sizeParser = htmlParser.size {
size = try? runHtmlComplexQuery(
@ -718,7 +737,7 @@ class ScrapingViewModel: ObservableObject {
let result = SearchResult(
title: title ?? "No title",
source: source.name,
source: subName.map { "\(source.name) - \($0)" } ?? source.name,
size: size ?? "",
magnet: Magnet(hash: nil, link: href),
seeders: seeders,