Creepso/Ferrite-backup
1
0
Fork 0
Code Issues 5 Pull requests Projects Releases 15 Packages Wiki Activity Actions

Sources: Add website API support

Browse source 
Adds support for website APIs both complex and simple. This commit
only supports GET requests to APIs. POST request support can be added
on request.

Client IDs and secrets are also supported. They can be added via
source settings or automatically set by a website endpoint.

Also fetch sources for scraping using the backgroundContext and remove
some functions from using the main actor.

Signed-off-by: kingbri <bdashore3@gmail.com>
This commit is contained in:
kingbri 2022-08-23 11:55:41 -04:00 committed by kingbri
parent 8c0e495f93
commit e0182a700f
19 changed files with 737 additions and 193 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

25
Ferrite.xcodeproj/project.pbxproj
View file

@ -9,6 +9,8 @@
/* Begin PBXBuildFile section */
0C0D50E5288DFE7F0035ECC8 /* SourceModels.swift in Sources */ = {isa = PBXBuildFile; fileRef = 0C0D50E4288DFE7F0035ECC8 /* SourceModels.swift */; };
0C0D50E7288DFF850035ECC8 /* SourcesView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 0C0D50E6288DFF850035ECC8 /* SourcesView.swift */; };
0C31133C28B1ABFA004DCB0D /* SourceJsonParser+CoreDataClass.swift in Sources */ = {isa = PBXBuildFile; fileRef = 0C31133A28B1ABFA004DCB0D /* SourceJsonParser+CoreDataClass.swift */; };
0C31133D28B1ABFA004DCB0D /* SourceJsonParser+CoreDataProperties.swift in Sources */ = {isa = PBXBuildFile; fileRef = 0C31133B28B1ABFA004DCB0D /* SourceJsonParser+CoreDataProperties.swift */; };
0C32FB532890D19D002BD219 /* AboutView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 0C32FB522890D19D002BD219 /* AboutView.swift */; };
0C32FB552890D1BF002BD219 /* UIApplication.swift in Sources */ = {isa = PBXBuildFile; fileRef = 0C32FB542890D1BF002BD219 /* UIApplication.swift */; };
0C32FB572890D1F2002BD219 /* ListRowViews.swift in Sources */ = {isa = PBXBuildFile; fileRef = 0C32FB562890D1F2002BD219 /* ListRowViews.swift */; };
@ -21,6 +23,7 @@
0C64A4B7288903880079976D /* KeychainSwift in Frameworks */ = {isa = PBXBuildFile; productRef = 0C64A4B6288903880079976D /* KeychainSwift */; };
0C733287289C4C820058D1FE /* SourceSettingsView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 0C733286289C4C820058D1FE /* SourceSettingsView.swift */; };
0C7376F028A97D1400D60918 /* SwiftUIX in Frameworks */ = {isa = PBXBuildFile; productRef = 0C7376EF28A97D1400D60918 /* SwiftUIX */; };
0C7506D728B1AC9A008BEE38 /* SwiftyJSON in Frameworks */ = {isa = PBXBuildFile; productRef = 0C7506D628B1AC9A008BEE38 /* SwiftyJSON */; };
0C750744289B003E004B3906 /* SourceRssParser+CoreDataClass.swift in Sources */ = {isa = PBXBuildFile; fileRef = 0C750742289B003E004B3906 /* SourceRssParser+CoreDataClass.swift */; };
0C750745289B003E004B3906 /* SourceRssParser+CoreDataProperties.swift in Sources */ = {isa = PBXBuildFile; fileRef = 0C750743289B003E004B3906 /* SourceRssParser+CoreDataProperties.swift */; };
0C794B67289DACB600DD1CC8 /* SourceUpdateButtonView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 0C794B66289DACB600DD1CC8 /* SourceUpdateButtonView.swift */; };
@ -72,6 +75,8 @@
/* Begin PBXFileReference section */
0C0D50E4288DFE7F0035ECC8 /* SourceModels.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SourceModels.swift; sourceTree = "<group>"; };
0C0D50E6288DFF850035ECC8 /* SourcesView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SourcesView.swift; sourceTree = "<group>"; };
0C31133A28B1ABFA004DCB0D /* SourceJsonParser+CoreDataClass.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "SourceJsonParser+CoreDataClass.swift"; sourceTree = "<group>"; };
0C31133B28B1ABFA004DCB0D /* SourceJsonParser+CoreDataProperties.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "SourceJsonParser+CoreDataProperties.swift"; sourceTree = "<group>"; };
0C32FB522890D19D002BD219 /* AboutView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AboutView.swift; sourceTree = "<group>"; };
0C32FB542890D1BF002BD219 /* UIApplication.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = UIApplication.swift; sourceTree = "<group>"; };
0C32FB562890D1F2002BD219 /* ListRowViews.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ListRowViews.swift; sourceTree = "<group>"; };
@ -134,6 +139,7 @@
isa = PBXFrameworksBuildPhase;
buildActionMask = 2147483647;
files = (
0C7506D728B1AC9A008BEE38 /* SwiftyJSON in Frameworks */,
0C64A4B4288903680079976D /* Base32 in Frameworks */,
0C4CFC462897030D00AD9FAD /* Regex in Frameworks */,
0C7376F028A97D1400D60918 /* SwiftUIX in Frameworks */,
@ -148,6 +154,8 @@
0C0D50DE288DF72D0035ECC8 /* Classes */ = {
isa = PBXGroup;
children = (
0C31133A28B1ABFA004DCB0D /* SourceJsonParser+CoreDataClass.swift */,
0C31133B28B1ABFA004DCB0D /* SourceJsonParser+CoreDataProperties.swift */,
0C750742289B003E004B3906 /* SourceRssParser+CoreDataClass.swift */,
0C750743289B003E004B3906 /* SourceRssParser+CoreDataProperties.swift */,
0C79DC052899AF3C003F1C5A /* SourceSeedLeech+CoreDataClass.swift */,
@ -342,6 +350,7 @@
0C64A4B6288903880079976D /* KeychainSwift */,
0C4CFC452897030D00AD9FAD /* Regex */,
0C7376EF28A97D1400D60918 /* SwiftUIX */,
0C7506D628B1AC9A008BEE38 /* SwiftyJSON */,
);
productName = Torrenter;
productReference = 0CAF1C68286F5C0E00296F86 /* Ferrite.app */;
@ -377,6 +386,7 @@
0C64A4B5288903880079976D /* XCRemoteSwiftPackageReference "keychain-swift" */,
0C4CFC442897030D00AD9FAD /* XCRemoteSwiftPackageReference "Regex" */,
0C7376EE28A97D1400D60918 /* XCRemoteSwiftPackageReference "SwiftUIX" */,
0C7506D528B1AC9A008BEE38 /* XCRemoteSwiftPackageReference "SwiftyJSON" */,
);
productRefGroup = 0CAF1C69286F5C0E00296F86 /* Products */;
projectDirPath = "";
@ -412,6 +422,7 @@
0CA148DB288903F000DE2211 /* NavView.swift in Sources */,
0C750745289B003E004B3906 /* SourceRssParser+CoreDataProperties.swift in Sources */,
0CBC7705288DE7F40054BE44 /* PersistenceController.swift in Sources */,
0C31133D28B1ABFA004DCB0D /* SourceJsonParser+CoreDataProperties.swift in Sources */,
0CA0545B288EEA4E00850554 /* SourceListEditorView.swift in Sources */,
0C84F4872895BFED0074B7C9 /* SourceList+CoreDataProperties.swift in Sources */,
0C794B6B289DACF100DD1CC8 /* SourceCatalogView.swift in Sources */,
@ -444,6 +455,7 @@
0C84F4772895BE680074B7C9 /* FerriteDB.xcdatamodeld in Sources */,
0C733287289C4C820058D1FE /* SourceSettingsView.swift in Sources */,
0CA05457288EE58200850554 /* SettingsSourceListView.swift in Sources */,
0C31133C28B1ABFA004DCB0D /* SourceJsonParser+CoreDataClass.swift in Sources */,
0CA148DE288903F000DE2211 /* RealDebridModels.swift in Sources */,
0CBC76FF288DAAD00054BE44 /* NavigationViewModel.swift in Sources */,
0C4CFC4D28970C8B00AD9FAD /* SourceComplexQuery+CoreDataClass.swift in Sources */,
@ -699,6 +711,14 @@
kind = branch;
};
};
0C7506D528B1AC9A008BEE38 /* XCRemoteSwiftPackageReference "SwiftyJSON" */ = {
isa = XCRemoteSwiftPackageReference;
repositoryURL = "https://github.com/SwiftyJSON/SwiftyJSON";
requirement = {
branch = master;
kind = branch;
};
};
0CAF1C79286F5C8600296F86 /* XCRemoteSwiftPackageReference "SwiftSoup" */ = {
isa = XCRemoteSwiftPackageReference;
repositoryURL = "https://github.com/scinfu/SwiftSoup.git";
@ -730,6 +750,11 @@
package = 0C7376EE28A97D1400D60918 /* XCRemoteSwiftPackageReference "SwiftUIX" */;
productName = SwiftUIX;
};
0C7506D628B1AC9A008BEE38 /* SwiftyJSON */ = {
isa = XCSwiftPackageProductDependency;
package = 0C7506D528B1AC9A008BEE38 /* XCRemoteSwiftPackageReference "SwiftyJSON" */;
productName = SwiftyJSON;
};
0CAF1C7A286F5C8600296F86 /* SwiftSoup */ = {
isa = XCSwiftPackageProductDependency;
package = 0CAF1C79286F5C8600296F86 /* XCRemoteSwiftPackageReference "SwiftSoup" */;

2
Ferrite/DataManagement/Classes/Source+CoreDataProperties.swift
View file

@ -25,7 +25,9 @@ public extension Source {
@NSManaged var version: Int16
@NSManaged var htmlParser: SourceHtmlParser?
@NSManaged var rssParser: SourceRssParser?
@NSManaged var jsonParser: SourceJsonParser?
@NSManaged var api: SourceApi?
@NSManaged var trackers: [String]?
}
extension Source: Identifiable {}

4
Ferrite/DataManagement/Classes/SourceComplexQuery+CoreDataProperties.swift
View file

@ -2,7 +2,7 @@
// SourceComplexQuery+CoreDataProperties.swift
// Ferrite
//
// Created by Brian Dashore on 7/31/22.
// Created by Brian Dashore on 8/22/22.
//
//
@ -15,7 +15,7 @@ public extension SourceComplexQuery {
}
@NSManaged var attribute: String
@NSManaged var lookupAttribute: String?
@NSManaged var discriminator: String?
@NSManaged var query: String
@NSManaged var regex: String?
}

32
Ferrite/DataManagement/Classes/SourceHtmlParser+CoreDataProperties.swift
View file

@ -6,28 +6,22 @@
//
//
import Foundation
import CoreData
import Foundation
extension SourceHtmlParser {
@nonobjc public class func fetchRequest() -> NSFetchRequest<SourceHtmlParser> {
return NSFetchRequest<SourceHtmlParser>(entityName: "SourceHtmlParser")
public extension SourceHtmlParser {
@nonobjc class func fetchRequest() -> NSFetchRequest<SourceHtmlParser> {
NSFetchRequest<SourceHtmlParser>(entityName: "SourceHtmlParser")
}
@NSManaged public var rows: String
@NSManaged public var searchUrl: String
@NSManaged public var trackers: [String]?
@NSManaged public var magnetHash: SourceMagnetHash?
@NSManaged public var magnetLink: SourceMagnetLink?
@NSManaged public var parentSource: Source?
@NSManaged public var seedLeech: SourceSeedLeech?
@NSManaged public var size: SourceSize?
@NSManaged public var title: SourceTitle?
@NSManaged var rows: String
@NSManaged var searchUrl: String
@NSManaged var magnetHash: SourceMagnetHash?
@NSManaged var magnetLink: SourceMagnetLink?
@NSManaged var parentSource: Source?
@NSManaged var seedLeech: SourceSeedLeech?
@NSManaged var size: SourceSize?
@NSManaged var title: SourceTitle?
}
extension SourceHtmlParser : Identifiable {
}
extension SourceHtmlParser: Identifiable {}

13
Ferrite/DataManagement/Classes/SourceJsonParser+CoreDataClass.swift Normal file
View file

@ -0,0 +1,13 @@
//
// SourceJsonParser+CoreDataClass.swift
// Ferrite
//
// Created by Brian Dashore on 8/20/22.
//
//
import CoreData
import Foundation
@objc(SourceJsonParser)
public class SourceJsonParser: NSManagedObject {}

28
Ferrite/DataManagement/Classes/SourceJsonParser+CoreDataProperties.swift Normal file
View file

@ -0,0 +1,28 @@
//
// SourceJsonParser+CoreDataProperties.swift
// Ferrite
//
// Created by Brian Dashore on 8/21/22.
//
//
import CoreData
import Foundation
public extension SourceJsonParser {
@nonobjc class func fetchRequest() -> NSFetchRequest<SourceJsonParser> {
NSFetchRequest<SourceJsonParser>(entityName: "SourceJsonParser")
}
@NSManaged var results: String?
@NSManaged var subResults: String?
@NSManaged var searchUrl: String
@NSManaged var magnetHash: SourceMagnetHash?
@NSManaged var magnetLink: SourceMagnetLink?
@NSManaged var parentSource: Source?
@NSManaged var seedLeech: SourceSeedLeech?
@NSManaged var size: SourceSize?
@NSManaged var title: SourceTitle?
}
extension SourceJsonParser: Identifiable {}

34
Ferrite/DataManagement/Classes/SourceRssParser+CoreDataProperties.swift
View file

@ -6,29 +6,23 @@
//
//
import Foundation
import CoreData
import Foundation
extension SourceRssParser {
@nonobjc public class func fetchRequest() -> NSFetchRequest<SourceRssParser> {
return NSFetchRequest<SourceRssParser>(entityName: "SourceRssParser")
public extension SourceRssParser {
@nonobjc class func fetchRequest() -> NSFetchRequest<SourceRssParser> {
NSFetchRequest<SourceRssParser>(entityName: "SourceRssParser")
}
@NSManaged public var items: String
@NSManaged public var rssUrl: String?
@NSManaged public var searchUrl: String
@NSManaged public var trackers: [String]?
@NSManaged public var magnetHash: SourceMagnetHash?
@NSManaged public var magnetLink: SourceMagnetLink?
@NSManaged public var parentSource: Source?
@NSManaged public var seedLeech: SourceSeedLeech?
@NSManaged public var size: SourceSize?
@NSManaged public var title: SourceTitle?
@NSManaged var items: String
@NSManaged var rssUrl: String?
@NSManaged var searchUrl: String
@NSManaged var magnetHash: SourceMagnetHash?
@NSManaged var magnetLink: SourceMagnetLink?
@NSManaged var parentSource: Source?
@NSManaged var seedLeech: SourceSeedLeech?
@NSManaged var size: SourceSize?
@NSManaged var title: SourceTitle?
}
extension SourceRssParser : Identifiable {
}
extension SourceRssParser: Identifiable {}

2
Ferrite/DataManagement/Classes/SourceSeedLeech+CoreDataProperties.swift
View file

@ -20,7 +20,7 @@ public extension SourceSeedLeech {
@NSManaged var seederRegex: String?
@NSManaged var seeders: String?
@NSManaged var attribute: String
@NSManaged var lookupAttribute: String?
@NSManaged var discriminator: String?
@NSManaged var parentParser: SourceHtmlParser?
}

47
Ferrite/DataManagement/FerriteDB.xcdatamodeld/FerriteDB.xcdatamodel/contents
View file

@ -9,34 +9,61 @@
<attribute name="listId" optional="YES" attributeType="UUID" usesScalarValueType="NO"/>
<attribute name="name" attributeType="String" defaultValueString=""/>
<attribute name="preferredParser" attributeType="Integer 16" defaultValueString="0" usesScalarValueType="YES"/>
<attribute name="trackers" optional="YES" attributeType="Transformable" valueTransformerName="NSSecureUnarchiveFromData" customClassName="[String]"/>
<attribute name="version" attributeType="Integer 16" defaultValueString="0" usesScalarValueType="YES"/>
<relationship name="api" optional="YES" maxCount="1" deletionRule="Cascade" destinationEntity="SourceApi" inverseName="parentSource" inverseEntity="SourceApi"/>
<relationship name="htmlParser" optional="YES" maxCount="1" deletionRule="Cascade" destinationEntity="SourceHtmlParser" inverseName="parentSource" inverseEntity="SourceHtmlParser"/>
<relationship name="jsonParser" optional="YES" maxCount="1" deletionRule="Cascade" destinationEntity="SourceJsonParser" inverseName="parentSource" inverseEntity="SourceJsonParser"/>
<relationship name="rssParser" optional="YES" maxCount="1" deletionRule="Cascade" destinationEntity="SourceRssParser" inverseName="parentSource" inverseEntity="SourceRssParser"/>
</entity>
<entity name="SourceApi" representedClassName="SourceApi" syncable="YES" codeGenerationType="class">
<attribute name="clientId" optional="YES" attributeType="String"/>
<attribute name="clientSecret" optional="YES" attributeType="String"/>
<attribute name="dynamicClientId" attributeType="Boolean" defaultValueString="NO" usesScalarValueType="YES"/>
<attribute name="apiUrl" optional="YES" attributeType="String"/>
<relationship name="clientId" optional="YES" maxCount="1" deletionRule="Cascade" destinationEntity="SourceApiClientId" inverseName="parentApi" inverseEntity="SourceApiClientId"/>
<relationship name="clientSecret" optional="YES" maxCount="1" deletionRule="Cascade" destinationEntity="SourceApiClientSecret" inverseName="parentApi" inverseEntity="SourceApiClientSecret"/>
<relationship name="parentSource" optional="YES" maxCount="1" deletionRule="Nullify" destinationEntity="Source" inverseName="api" inverseEntity="Source"/>
</entity>
<entity name="SourceApiClientId" representedClassName="SourceApiClientId" parentEntity="SourceApiCredential" syncable="YES" codeGenerationType="class">
<relationship name="parentApi" optional="YES" maxCount="1" deletionRule="Nullify" destinationEntity="SourceApi" inverseName="clientId" inverseEntity="SourceApi"/>
</entity>
<entity name="SourceApiClientSecret" representedClassName="SourceApiClientSecret" parentEntity="SourceApiCredential" syncable="YES" codeGenerationType="class">
<relationship name="parentApi" optional="YES" maxCount="1" deletionRule="Nullify" destinationEntity="SourceApi" inverseName="clientSecret" inverseEntity="SourceApi"/>
</entity>
<entity name="SourceApiCredential" representedClassName="SourceApiCredential" isAbstract="YES" syncable="YES" codeGenerationType="class">
<attribute name="dynamic" attributeType="Boolean" defaultValueString="NO" usesScalarValueType="YES"/>
<attribute name="expiryLength" optional="YES" attributeType="Double" defaultValueString="0" usesScalarValueType="YES"/>
<attribute name="query" optional="YES" attributeType="String"/>
<attribute name="responseType" optional="YES" attributeType="String"/>
<attribute name="timeStamp" optional="YES" attributeType="Date" usesScalarValueType="NO"/>
<attribute name="urlString" optional="YES" attributeType="String"/>
<attribute name="value" optional="YES" attributeType="String"/>
</entity>
<entity name="SourceComplexQuery" representedClassName="SourceComplexQuery" isAbstract="YES" syncable="YES">
<attribute name="attribute" attributeType="String" defaultValueString="text"/>
<attribute name="lookupAttribute" optional="YES" attributeType="String"/>
<attribute name="discriminator" optional="YES" attributeType="String"/>
<attribute name="query" attributeType="String" defaultValueString=""/>
<attribute name="regex" optional="YES" attributeType="String"/>
</entity>
<entity name="SourceHtmlParser" representedClassName="SourceHtmlParser" syncable="YES">
<attribute name="rows" attributeType="String" defaultValueString=""/>
<attribute name="searchUrl" attributeType="String" defaultValueString=""/>
<attribute name="trackers" optional="YES" attributeType="Transformable" valueTransformerName="NSSecureUnarchiveFromData" customClassName="[String]"/>
<relationship name="magnetHash" optional="YES" maxCount="1" deletionRule="Cascade" destinationEntity="SourceMagnetHash" inverseName="parentHtmlParser" inverseEntity="SourceMagnetHash"/>
<relationship name="magnetLink" optional="YES" maxCount="1" deletionRule="Cascade" destinationEntity="SourceMagnetLink" inverseName="parentHtmlParser" inverseEntity="SourceMagnetLink"/>
<relationship name="parentSource" optional="YES" maxCount="1" deletionRule="Nullify" destinationEntity="Source" inverseName="htmlParser" inverseEntity="Source"/>
<relationship name="seedLeech" optional="YES" maxCount="1" deletionRule="Nullify" destinationEntity="SourceSeedLeech" inverseName="parentHtmlParser" inverseEntity="SourceSeedLeech"/>
<relationship name="seedLeech" optional="YES" maxCount="1" deletionRule="Cascade" destinationEntity="SourceSeedLeech" inverseName="parentHtmlParser" inverseEntity="SourceSeedLeech"/>
<relationship name="size" optional="YES" maxCount="1" deletionRule="Cascade" destinationEntity="SourceSize" inverseName="parentHtmlParser" inverseEntity="SourceSize"/>
<relationship name="title" optional="YES" maxCount="1" deletionRule="Cascade" destinationEntity="SourceTitle" inverseName="parentHtmlParser" inverseEntity="SourceTitle"/>
</entity>
<entity name="SourceJsonParser" representedClassName="SourceJsonParser" syncable="YES">
<attribute name="results" optional="YES" attributeType="String" valueTransformerName="NSSecureUnarchiveFromData" customClassName="[String]"/>
<attribute name="searchUrl" optional="YES" attributeType="String"/>
<attribute name="subResults" optional="YES" attributeType="String"/>
<relationship name="magnetHash" optional="YES" maxCount="1" deletionRule="Cascade" destinationEntity="SourceMagnetHash" inverseName="parentJsonParser" inverseEntity="SourceMagnetHash"/>
<relationship name="magnetLink" optional="YES" maxCount="1" deletionRule="Cascade" destinationEntity="SourceMagnetLink" inverseName="parentJsonParser" inverseEntity="SourceMagnetLink"/>
<relationship name="parentSource" optional="YES" maxCount="1" deletionRule="Nullify" destinationEntity="Source" inverseName="jsonParser" inverseEntity="Source"/>
<relationship name="seedLeech" optional="YES" maxCount="1" deletionRule="Cascade" destinationEntity="SourceSeedLeech" inverseName="parentJsonParser" inverseEntity="SourceSeedLeech"/>
<relationship name="size" optional="YES" maxCount="1" deletionRule="Cascade" destinationEntity="SourceSize" inverseName="parentJsonParser" inverseEntity="SourceSize"/>
<relationship name="title" optional="YES" maxCount="1" deletionRule="Cascade" destinationEntity="SourceTitle" inverseName="parentJsonParser" inverseEntity="SourceTitle"/>
</entity>
<entity name="SourceList" representedClassName="SourceList" syncable="YES">
<attribute name="author" attributeType="String" defaultValueString=""/>
<attribute name="id" attributeType="UUID" usesScalarValueType="NO"/>
@ -45,18 +72,19 @@
</entity>
<entity name="SourceMagnetHash" representedClassName="SourceMagnetHash" parentEntity="SourceComplexQuery" syncable="YES" codeGenerationType="class">
<relationship name="parentHtmlParser" optional="YES" maxCount="1" deletionRule="Nullify" destinationEntity="SourceHtmlParser" inverseName="magnetHash" inverseEntity="SourceHtmlParser"/>
<relationship name="parentJsonParser" optional="YES" maxCount="1" deletionRule="Nullify" destinationEntity="SourceJsonParser" inverseName="magnetHash" inverseEntity="SourceJsonParser"/>
<relationship name="parentRssParser" optional="YES" maxCount="1" deletionRule="Nullify" destinationEntity="SourceRssParser" inverseName="magnetHash" inverseEntity="SourceRssParser"/>
</entity>
<entity name="SourceMagnetLink" representedClassName="SourceMagnetLink" parentEntity="SourceComplexQuery" syncable="YES" codeGenerationType="class">
<attribute name="externalLinkQuery" optional="YES" attributeType="String"/>
<relationship name="parentHtmlParser" optional="YES" maxCount="1" deletionRule="Nullify" destinationEntity="SourceHtmlParser" inverseName="magnetLink" inverseEntity="SourceHtmlParser"/>
<relationship name="parentJsonParser" optional="YES" maxCount="1" deletionRule="Nullify" destinationEntity="SourceJsonParser" inverseName="magnetLink" inverseEntity="SourceJsonParser"/>
<relationship name="parentRssParser" optional="YES" maxCount="1" deletionRule="Nullify" destinationEntity="SourceRssParser" inverseName="magnetLink" inverseEntity="SourceRssParser"/>
</entity>
<entity name="SourceRssParser" representedClassName="SourceRssParser" syncable="YES">
<attribute name="items" attributeType="String" defaultValueString=""/>
<attribute name="rssUrl" optional="YES" attributeType="String"/>
<attribute name="searchUrl" attributeType="String" defaultValueString=""/>
<attribute name="trackers" optional="YES" attributeType="Transformable" valueTransformerName="NSSecureUnarchiveFromData" customClassName="[String]"/>
<relationship name="magnetHash" optional="YES" maxCount="1" deletionRule="Cascade" destinationEntity="SourceMagnetHash" inverseName="parentRssParser" inverseEntity="SourceMagnetHash"/>
<relationship name="magnetLink" optional="YES" maxCount="1" deletionRule="Cascade" destinationEntity="SourceMagnetLink" inverseName="parentRssParser" inverseEntity="SourceMagnetLink"/>
<relationship name="parentSource" optional="YES" maxCount="1" deletionRule="Nullify" destinationEntity="Source" inverseName="rssParser" inverseEntity="Source"/>
@ -67,20 +95,23 @@
<entity name="SourceSeedLeech" representedClassName="SourceSeedLeech" syncable="YES">
<attribute name="attribute" attributeType="String" defaultValueString=""/>
<attribute name="combined" optional="YES" attributeType="String"/>
<attribute name="discriminator" optional="YES" attributeType="String"/>
<attribute name="leecherRegex" optional="YES" attributeType="String"/>
<attribute name="leechers" optional="YES" attributeType="String"/>
<attribute name="lookupAttribute" optional="YES" attributeType="String"/>
<attribute name="seederRegex" optional="YES" attributeType="String"/>
<attribute name="seeders" optional="YES" attributeType="String"/>
<relationship name="parentHtmlParser" optional="YES" maxCount="1" deletionRule="Nullify" destinationEntity="SourceHtmlParser" inverseName="seedLeech" inverseEntity="SourceHtmlParser"/>
<relationship name="parentJsonParser" optional="YES" maxCount="1" deletionRule="Nullify" destinationEntity="SourceJsonParser" inverseName="seedLeech" inverseEntity="SourceJsonParser"/>
<relationship name="parentRssParser" optional="YES" maxCount="1" deletionRule="Nullify" destinationEntity="SourceRssParser" inverseName="seedLeech" inverseEntity="SourceRssParser"/>
</entity>
<entity name="SourceSize" representedClassName="SourceSize" parentEntity="SourceComplexQuery" syncable="YES" codeGenerationType="class">
<relationship name="parentHtmlParser" optional="YES" maxCount="1" deletionRule="Nullify" destinationEntity="SourceHtmlParser" inverseName="size" inverseEntity="SourceHtmlParser"/>
<relationship name="parentJsonParser" optional="YES" maxCount="1" deletionRule="Nullify" destinationEntity="SourceJsonParser" inverseName="size" inverseEntity="SourceJsonParser"/>
<relationship name="parentRssParser" optional="YES" maxCount="1" deletionRule="Nullify" destinationEntity="SourceRssParser" inverseName="size" inverseEntity="SourceRssParser"/>
</entity>
<entity name="SourceTitle" representedClassName="SourceTitle" parentEntity="SourceComplexQuery" syncable="YES" codeGenerationType="class">
<relationship name="parentHtmlParser" optional="YES" maxCount="1" deletionRule="Nullify" destinationEntity="SourceHtmlParser" inverseName="title" inverseEntity="SourceHtmlParser"/>
<relationship name="parentJsonParser" optional="YES" maxCount="1" deletionRule="Nullify" destinationEntity="SourceJsonParser" inverseName="title" inverseEntity="SourceJsonParser"/>
<relationship name="parentRssParser" optional="YES" maxCount="1" deletionRule="Nullify" destinationEntity="SourceRssParser" inverseName="title" inverseEntity="SourceRssParser"/>
</entity>
</model>

38
Ferrite/Models/SourceModels.swift
View file

@ -7,6 +7,11 @@
import Foundation
public enum ApiCredentialResponseType: String, Codable, Hashable {
case json
case text
}
public struct SourceListJson: Codable {
let name: String
let author: String
@ -20,7 +25,9 @@ public struct SourceJson: Codable, Hashable {
var dynamicBaseUrl: Bool?
var author: String?
var listId: UUID?
let trackers: [String]?
let api: SourceApiJson?
let jsonParser: SourceJsonParserJson?
let rssParser: SourceRssParserJson?
let htmlParser: SourceHtmlParserJson?
}
@ -33,9 +40,29 @@ public enum SourcePreferredParser: Int16, CaseIterable {
}
public struct SourceApiJson: Codable, Hashable {
let clientId: String?
var dynamicClientId: Bool?
let usesSecret: Bool
let apiUrl: String?
let clientId: SourceApiCredentialJson?
let clientSecret: SourceApiCredentialJson?
}
public struct SourceApiCredentialJson: Codable, Hashable {
let query: String?
let value: String?
let dynamic: Bool?
let url: String?
let responseType: ApiCredentialResponseType?
let expiryLength: Double?
}
public struct SourceJsonParserJson: Codable, Hashable {
let searchUrl: String
let results: String?
let subResults: String?
let magnetHash: SouceComplexQueryJson?
let magnetLink: SouceComplexQueryJson?
let title: SouceComplexQueryJson?
let size: SouceComplexQueryJson?
let sl: SourceSLJson?
}
public struct SourceRssParserJson: Codable, Hashable {
@ -47,7 +74,6 @@ public struct SourceRssParserJson: Codable, Hashable {
let title: SouceComplexQueryJson?
let size: SouceComplexQueryJson?
let sl: SourceSLJson?
let trackers: [String]?
}
public struct SourceHtmlParserJson: Codable, Hashable {
@ -61,7 +87,7 @@ public struct SourceHtmlParserJson: Codable, Hashable {
public struct SouceComplexQueryJson: Codable, Hashable {
let query: String
let lookupAttribute: String?
let discriminator: String?
let attribute: String?
let regex: String?
}
@ -78,7 +104,7 @@ public struct SourceSLJson: Codable, Hashable {
let leechers: String?
let combined: String?
let attribute: String?
let lookupAttribute: String?
let discriminator: String?
let seederRegex: String?
let leecherRegex: String?
}

11
Ferrite/ViewModels/DebridManager.swift
View file

@ -106,8 +106,17 @@ public class DebridManager: ObservableObject {
}
public func fetchRdDownload(searchResult: SearchResult, iaFile: RealDebridIAFile? = nil) async {
guard let magnetLink = searchResult.magnetLink else {
Task { @MainActor in
toastModel?.toastDescription = "Could not run your action because the magnet link is invalid."
}
print("RD error: Invalid magnet link")
return
}
do {
let realDebridId = try await realDebrid.addMagnet(magnetLink: searchResult.magnetLink)
let realDebridId = try await realDebrid.addMagnet(magnetLink: magnetLink)
var fileIds: [Int] = []

13
Ferrite/ViewModels/NavigationViewModel.swift
View file

@ -86,17 +86,26 @@ class NavigationViewModel: ObservableObject {
public func runMagnetAction(action: DefaultMagnetActionType?, searchResult: SearchResult) {
let selectedAction = action ?? defaultMagnetAction
guard let magnetLink = searchResult.magnetLink else {
toastModel?.toastDescription = "Could not run your action because the magnet link is invalid."
print("Magnet action error: The magnet link is invalid.")
return
}
switch selectedAction {
case .none:
currentChoiceSheet = .magnet
case .webtor:
if let url = URL(string: "https://webtor.io/#/show?magnet=\(searchResult.magnetLink)") {
if let url = URL(string: "https://webtor.io/#/show?magnet=\(magnetLink)") {
UIApplication.shared.open(url)
} else {
toastModel?.toastDescription = "Could not create a WebTor URL"
}
case .shareMagnet:
if let magnetUrl = URL(string: searchResult.magnetLink), currentChoiceSheet == nil {
if let magnetUrl = URL(string: magnetLink),
currentChoiceSheet == nil
{
activityItems = [magnetUrl]
showActivityView.toggle()
} else {

507
Ferrite/ViewModels/ScrapingViewModel.swift
View file

@ -9,12 +9,13 @@ import Base32
import Regex
import SwiftSoup
import SwiftUI
import SwiftyJSON
public struct SearchResult: Hashable, Codable {
let title: String
let title: String?
let source: String
let size: String
let magnetLink: String
let size: String?
let magnetLink: String?
let magnetHash: String?
let seeders: String?
let leechers: String?
@ -42,7 +43,7 @@ class ScrapingViewModel: ObservableObject {
toastModel?.toastDescription = "There are no sources to search!"
}
print("Sources empty")
print("There are no sources to search!")
return
}
@ -53,9 +54,7 @@ class ScrapingViewModel: ObservableObject {
currentSourceName = source.name
guard let baseUrl = source.baseUrl else {
Task { @MainActor in
toastModel?.toastDescription = "The base URL could not be found for source \(source.name)"
}
toastModel?.toastDescription = "The base URL could not be found for source \(source.name)"
print("The base URL could not be found for source \(source.name)")
continue
@ -64,54 +63,82 @@ class ScrapingViewModel: ObservableObject {
// Default to HTML scraping
let preferredParser = SourcePreferredParser(rawValue: source.preferredParser) ?? .none
guard let encodedQuery = searchText.addingPercentEncoding(withAllowedCharacters: .urlPathAllowed) else {
toastModel?.toastDescription = "Could not process search query, invalid characters present."
print("Could not process search query, invalid characters present")
continue
}
switch preferredParser {
case .scraping:
if let htmlParser = source.htmlParser {
guard let encodedQuery = searchText.addingPercentEncoding(withAllowedCharacters: .urlPathAllowed) else {
toastModel?.toastDescription = "Could not process search query, invalid characters present."
print("Could not process search query, invalid characters present")
let urlString = baseUrl + htmlParser.searchUrl
.replacingOccurrences(of: "{query}", with: encodedQuery)
continue
if let data = await fetchWebsiteData(urlString: urlString),
let html = String(data: data, encoding: .utf8)
{
let sourceResults = await scrapeHtml(source: source, baseUrl: baseUrl, html: html)
tempResults += sourceResults
}
let urlString = baseUrl + htmlParser.searchUrl.replacingOccurrences(of: "{query}", with: encodedQuery)
guard let html = await fetchWebsiteData(urlString: urlString) else {
continue
}
let sourceResults = await scrapeHtml(source: source, baseUrl: baseUrl, html: html)
tempResults += sourceResults
}
case .rss:
if let rssParser = source.rssParser {
guard let encodedQuery = searchText.addingPercentEncoding(withAllowedCharacters: .urlPathAllowed) else {
toastModel?.toastDescription = "Could not process search query, invalid characters present."
print("Could not process search query, invalid characters present")
continue
}
let replacedSearchUrl = rssParser.searchUrl
.replacingOccurrences(of: "{apiKey}", with: source.api?.clientSecret ?? "")
.replacingOccurrences(of: "{secret}", with: source.api?.clientSecret?.value ?? "")
.replacingOccurrences(of: "{query}", with: encodedQuery)
// If there is an RSS base URL, use that instead
var urlString: String
if let rssUrl = rssParser.rssUrl {
urlString = rssUrl + replacedSearchUrl
} else {
urlString = baseUrl + replacedSearchUrl
}
let urlString = (rssParser.rssUrl ?? baseUrl) + replacedSearchUrl
guard let rss = await fetchWebsiteData(urlString: urlString) else {
continue
if let data = await fetchWebsiteData(urlString: urlString),
let rss = String(data: data, encoding: .utf8)
{
let sourceResults = scrapeRss(source: source, rss: rss)
tempResults += sourceResults
}
let sourceResults = scrapeRss(source: source, rss: rss)
tempResults += sourceResults
}
case .siteApi, .none:
case .siteApi:
if let jsonParser = source.jsonParser {
var replacedSearchUrl = jsonParser.searchUrl
.replacingOccurrences(of: "{query}", with: encodedQuery)
// Handle anything API related including tokens, client IDs, and appending the API URL
// The source API key is for APIs that require extra credentials or use a different URL
if let sourceApi = source.api {
if let clientIdInfo = sourceApi.clientId {
if let newSearchUrl = await handleApiCredential(clientIdInfo,
replacement: "{clientId}",
searchUrl: replacedSearchUrl,
apiUrl: sourceApi.apiUrl,
baseUrl: baseUrl)
{
replacedSearchUrl = newSearchUrl
}
}
// Works exactly the same as the client ID check
if let clientSecretInfo = sourceApi.clientSecret {
if let newSearchUrl = await handleApiCredential(clientSecretInfo,
replacement: "{secret}",
searchUrl: replacedSearchUrl,
apiUrl: sourceApi.apiUrl,
baseUrl: baseUrl)
{
replacedSearchUrl = newSearchUrl
}
}
}
let urlString = (source.api?.apiUrl ?? baseUrl) + replacedSearchUrl
if let data = await fetchWebsiteData(urlString: urlString) {
let sourceResults = scrapeJson(source: source, jsonData: data)
tempResults += sourceResults
}
}
case .none:
continue
}
}
@ -125,11 +152,90 @@ class ScrapingViewModel: ObservableObject {
searchResults = tempResults
}
// Fetches the data for a URL
@MainActor
public func fetchWebsiteData(urlString: String) async -> String? {
public func handleApiCredential(_ credential: SourceApiCredential,
replacement: String,
searchUrl: String,
apiUrl: String?,
baseUrl: String) async -> String?
{
// Is the credential expired
var isExpired = false
if let timeStamp = credential.timeStamp?.timeIntervalSince1970, credential.expiryLength != 0 {
let now = Date().timeIntervalSince1970
isExpired = now > timeStamp + credential.expiryLength
}
// Fetch a new credential if it's expired or doesn't exist yet
if let value = credential.value, !isExpired {
return searchUrl
.replacingOccurrences(of: replacement, with: value)
} else if
credential.value == nil || isExpired,
let credentialUrl = credential.urlString,
let newValue = await fetchApiCredential(
urlString: (apiUrl ?? baseUrl) + credentialUrl,
credential: credential
)
{
let backgroundContext = PersistenceController.shared.backgroundContext
credential.value = newValue
credential.timeStamp = Date()
PersistenceController.shared.save(backgroundContext)
return searchUrl
.replacingOccurrences(of: replacement, with: newValue)
}
return nil
}
public func fetchApiCredential(urlString: String, credential: SourceApiCredential) async -> String? {
guard let url = URL(string: urlString) else {
toastModel?.toastDescription = "Source doesn't contain a valid URL, contact the source dev!"
Task { @MainActor in
toastModel?.toastDescription = "This token URL is invalid."
}
print("Token url \(urlString) is invalid!")
return nil
}
do {
let (data, _) = try await URLSession.shared.data(from: url)
let responseType = ApiCredentialResponseType(rawValue: credential.responseType ?? "") ?? .json
switch responseType {
case .json:
guard let credentialQuery = credential.query else {
return nil
}
let json = try JSON(data: data)
return json[credentialQuery.components(separatedBy: ".")].string
case .text:
return String(data: data, encoding: .utf8)
}
} catch {
Task { @MainActor in
toastModel?.toastDescription = "Error in fetching an API credential \(error)"
}
print("Error in fetching an API credential \(error)")
return nil
}
}
// Fetches the data for a URL
public func fetchWebsiteData(urlString: String) async -> Data? {
guard let url = URL(string: urlString) else {
Task { @MainActor in
toastModel?.toastDescription = "Source doesn't contain a valid URL, contact the source dev!"
}
print("Source doesn't contain a valid URL, contact the source dev!")
return nil
@ -137,27 +243,182 @@ class ScrapingViewModel: ObservableObject {
do {
let (data, _) = try await URLSession.shared.data(from: url)
let html = String(data: data, encoding: .ascii)
return html
return data
} catch {
let error = error as NSError
switch error.code {
case -999:
toastModel?.toastType = .info
toastModel?.toastDescription = "Search cancelled"
default:
toastModel?.toastDescription = "Error in fetching data \(error)"
Task { @MainActor in
switch error.code {
case -999:
toastModel?.toastType = .info
toastModel?.toastDescription = "Search cancelled"
default:
toastModel?.toastDescription = "Error in fetching data \(error)"
}
}
print("Error in fetching data \(error)")
return nil
}
}
public func scrapeJson(source: Source, jsonData: Data) -> [SearchResult] {
var tempResults: [SearchResult] = []
guard let jsonParser = source.jsonParser else {
return tempResults
}
var jsonResults: [JSON] = []
do {
let json = try JSON(data: jsonData)
if let resultsQuery = jsonParser.results {
jsonResults = json[resultsQuery.components(separatedBy: ".")].arrayValue
} else {
jsonResults = json.arrayValue
}
} catch {
if let api = source.api {
Task { @MainActor in
cleanApiCreds(api: api)
}
print("JSON parsing error, couldn't fetch results: \(error)")
}
}
// If there are no results and the client secret isn't dynamic, just clear out the token
if let api = source.api, jsonResults.isEmpty {
Task { @MainActor in
cleanApiCreds(api: api)
}
print("JSON results were empty!")
}
// Iterate through results and grab what we can
for result in jsonResults {
var subResults: [JSON] = []
let searchResult = parseJsonResult(result, jsonParser: jsonParser, source: source)
// If subresults exist, iterate through those as well with the existing result
// Otherwise append the applied result if it exists
// Better to be redundant with checks rather than another for loop or filter
if let subResultsQuery = jsonParser.subResults {
subResults = result[subResultsQuery.components(separatedBy: ".")].arrayValue
for subResult in subResults {
if let newSearchResult =
parseJsonResult(
subResult,
jsonParser: jsonParser,
source: source,
existingSearchResult: searchResult
),
let magnetLink = newSearchResult.magnetLink,
magnetLink.starts(with: "magnet:"),
!tempResults.contains(newSearchResult)
{
tempResults.append(newSearchResult)
}
}
} else if
let searchResult = searchResult,
let magnetLink = searchResult.magnetLink,
magnetLink.starts(with: "magnet:"),
!tempResults.contains(searchResult)
{
tempResults.append(searchResult)
}
}
return tempResults
}
public func parseJsonResult(_ result: JSON, jsonParser: SourceJsonParser, source: Source, existingSearchResult: SearchResult? = nil) -> SearchResult? {
var magnetHash: String? = existingSearchResult?.magnetHash
if let magnetHashParser = jsonParser.magnetHash {
let rawHash = result[magnetHashParser.query.components(separatedBy: ".")].rawValue
if !(rawHash is NSNull) {
magnetHash = fetchMagnetHash(existingHash: String(describing: rawHash))
}
}
var title: String? = existingSearchResult?.title
if let titleParser = jsonParser.title {
if let existingTitle = existingSearchResult?.title,
let discriminatorQuery = titleParser.discriminator
{
let rawDiscriminator = result[discriminatorQuery.components(separatedBy: ".")].rawValue
if !(rawDiscriminator is NSNull) {
title = String(describing: rawDiscriminator) + existingTitle
}
} else if existingSearchResult?.title == nil {
let rawTitle = result[titleParser.query].rawValue
title = rawTitle is NSNull ? nil : String(describing: rawTitle)
}
}
var link: String? = existingSearchResult?.magnetLink
if let magnetLinkParser = jsonParser.magnetLink, existingSearchResult?.magnetLink == nil {
let rawLink = result[magnetLinkParser.query.components(separatedBy: ".")].rawValue
link = rawLink is NSNull ? nil : String(describing: rawLink)
} else if let magnetHash = magnetHash {
link = generateMagnetLink(magnetHash: magnetHash, title: title, trackers: source.trackers)
}
if magnetHash == nil, let href = link {
magnetHash = fetchMagnetHash(magnetLink: href)
}
var size: String? = existingSearchResult?.size
if let sizeParser = jsonParser.size, existingSearchResult?.size == nil {
let rawSize = result[sizeParser.query.components(separatedBy: ".")].rawValue
size = rawSize is NSNull ? nil : String(describing: rawSize)
}
if let sizeString = size, let sizeInt = Int64(sizeString) {
size = byteCountFormatter.string(fromByteCount: sizeInt)
}
var seeders: String? = existingSearchResult?.seeders
var leechers: String? = existingSearchResult?.leechers
if let seederLeecher = jsonParser.seedLeech {
if let seederQuery = seederLeecher.seeders, existingSearchResult?.seeders == nil {
let rawSeeders = result[seederQuery.components(separatedBy: ".")].rawValue
seeders = rawSeeders is NSNull ? nil : String(describing: rawSeeders)
}
if let leecherQuery = seederLeecher.leechers, existingSearchResult?.leechers == nil {
let rawLeechers = result[leecherQuery.components(separatedBy: ".")].rawValue
leechers = rawLeechers is NSNull ? nil : String(describing: rawLeechers)
}
}
let result = SearchResult(
title: title,
source: source.name,
size: size,
magnetLink: link,
magnetHash: magnetHash,
seeders: seeders,
leechers: leechers
)
return result
}
// RSS feed scraper
@MainActor
public func scrapeRss(source: Source, rss: String) -> [SearchResult] {
var tempResults: [SearchResult] = []
@ -171,7 +432,9 @@ class ScrapingViewModel: ObservableObject {
let document = try SwiftSoup.parse(rss, "", Parser.xmlParser())
items = try document.getElementsByTag("item")
} catch {
toastModel?.toastDescription = "RSS scraping error, couldn't fetch items: \(error)"
Task { @MainActor in
toastModel?.toastDescription = "RSS scraping error, couldn't fetch items: \(error)"
}
print("RSS scraping error, couldn't fetch items: \(error)")
return tempResults
@ -181,13 +444,15 @@ class ScrapingViewModel: ObservableObject {
// Parse magnet link or translate hash
var magnetHash: String?
if let magnetHashParser = rssParser.magnetHash {
magnetHash = try? runRssComplexQuery(
let tempHash = try? runRssComplexQuery(
item: item,
query: magnetHashParser.query,
attribute: magnetHashParser.attribute,
lookupAttribute: magnetHashParser.lookupAttribute,
discriminator: magnetHashParser.discriminator,
regexString: magnetHashParser.regex
)
magnetHash = fetchMagnetHash(existingHash: tempHash)
}
var title: String?
@ -196,7 +461,7 @@ class ScrapingViewModel: ObservableObject {
item: item,
query: titleParser.query,
attribute: titleParser.attribute,
lookupAttribute: titleParser.lookupAttribute,
discriminator: titleParser.discriminator,
regexString: titleParser.regex
)
}
@ -207,11 +472,11 @@ class ScrapingViewModel: ObservableObject {
item: item,
query: magnetLinkParser.query,
attribute: magnetLinkParser.attribute,
lookupAttribute: magnetLinkParser.lookupAttribute,
discriminator: magnetLinkParser.discriminator,
regexString: magnetLinkParser.regex
)
} else if let magnetHash = magnetHash {
link = generateMagnetLink(magnetHash: magnetHash, title: title, trackers: rssParser.trackers)
link = generateMagnetLink(magnetHash: magnetHash, title: title, trackers: source.trackers)
} else {
continue
}
@ -230,7 +495,7 @@ class ScrapingViewModel: ObservableObject {
item: item,
query: sizeParser.query,
attribute: sizeParser.attribute,
lookupAttribute: sizeParser.lookupAttribute,
discriminator: sizeParser.discriminator,
regexString: sizeParser.regex
)
}
@ -247,7 +512,7 @@ class ScrapingViewModel: ObservableObject {
item: item,
query: seederQuery,
attribute: seederLeecher.attribute,
lookupAttribute: seederLeecher.lookupAttribute,
discriminator: seederLeecher.discriminator,
regexString: seederLeecher.seederRegex
)
}
@ -257,7 +522,7 @@ class ScrapingViewModel: ObservableObject {
item: item,
query: leecherQuery,
attribute: seederLeecher.attribute,
lookupAttribute: seederLeecher.lookupAttribute,
discriminator: seederLeecher.discriminator,
regexString: seederLeecher.leecherRegex
)
}
@ -281,8 +546,36 @@ class ScrapingViewModel: ObservableObject {
return tempResults
}
// Complex query parsing for RSS scraping
func runRssComplexQuery(item: Element, query: String, attribute: String, discriminator: String?, regexString: String?) throws -> String? {
var parsedValue: String?
switch attribute {
case "text":
parsedValue = try item.getElementsByTag(query).first()?.text()
default:
// If there's a key/value to lookup the attribute with, query it. Othewise assume the value is in the same attribute
if let discriminator = discriminator {
let containerElement = try item.getElementsByAttributeValue(discriminator, query).first()
parsedValue = try containerElement?.attr(attribute)
} else {
let containerElement = try item.getElementsByAttribute(attribute).first()
parsedValue = try containerElement?.attr(attribute)
}
}
// A capture group must be used in the provided regex
if let regexString = regexString,
let parsedValue = parsedValue,
let regexValue = try? Regex(regexString).firstMatch(in: parsedValue)?.groups[safe: 0]?.value
{
return regexValue
} else {
return parsedValue
}
}
// HTML scraper
@MainActor
public func scrapeHtml(source: Source, baseUrl: String, html: String) async -> [SearchResult] {
var tempResults: [SearchResult] = []
@ -296,7 +589,9 @@ class ScrapingViewModel: ObservableObject {
let document = try SwiftSoup.parse(html)
rows = try document.select(htmlParser.rows)
} catch {
toastModel?.toastDescription = "Scraping error, couldn't fetch rows: \(error)"
Task { @MainActor in
toastModel?.toastDescription = "Scraping error, couldn't fetch rows: \(error)"
}
print("Scraping error, couldn't fetch rows: \(error)")
return tempResults
@ -314,11 +609,11 @@ class ScrapingViewModel: ObservableObject {
var href: String
if let externalMagnetQuery = magnetParser.externalLinkQuery, !externalMagnetQuery.isEmpty {
guard let externalMagnetLink = try row.select(externalMagnetQuery).first()?.attr("href") else {
continue
}
guard let magnetHtml = await fetchWebsiteData(urlString: baseUrl + externalMagnetLink) else {
guard
let externalMagnetLink = try row.select(externalMagnetQuery).first()?.attr("href"),
let data = await fetchWebsiteData(urlString: baseUrl + externalMagnetLink),
let magnetHtml = String(data: data, encoding: .utf8)
else {
continue
}
@ -429,7 +724,9 @@ class ScrapingViewModel: ObservableObject {
tempResults.append(result)
}
} catch {
toastModel?.toastDescription = "Scraping error: \(error)"
Task { @MainActor in
toastModel?.toastDescription = "Scraping error: \(error)"
}
print("Scraping error: \(error)")
continue
@ -463,42 +760,19 @@ class ScrapingViewModel: ObservableObject {
}
}
// Complex query parsing for RSS scraping
func runRssComplexQuery(item: Element, query: String, attribute: String, lookupAttribute: String?, regexString: String?) throws -> String? {
var parsedValue: String?
switch attribute {
case "text":
parsedValue = try item.getElementsByTag(query).first()?.text()
default:
// If there's a key/value to lookup the attribute with, query it. Othewise assume the value is in the same attribute
if let lookupAttribute = lookupAttribute {
let containerElement = try item.getElementsByAttributeValue(lookupAttribute, query).first()
parsedValue = try containerElement?.attr(attribute)
} else {
let containerElement = try item.getElementsByAttribute(attribute).first()
parsedValue = try containerElement?.attr(attribute)
}
}
// A capture group must be used in the provided regex
if let regexString = regexString,
let parsedValue = parsedValue,
let regexValue = try? Regex(regexString).firstMatch(in: parsedValue)?.groups[safe: 0]?.value
{
return regexValue
} else {
return parsedValue
}
}
// Fetches and possibly converts the magnet hash value to sha1
public func fetchMagnetHash(magnetLink: String) -> String? {
guard let firstSplit = magnetLink.split(separator: ":")[safe: 3] else {
return nil
}
public func fetchMagnetHash(magnetLink: String? = nil, existingHash: String? = nil) -> String? {
var magnetHash: String
guard let magnetHash = firstSplit.split(separator: "&")[safe: 0] else {
if let existingHash = existingHash {
magnetHash = existingHash
} else if
let magnetLink = magnetLink,
let firstSplit = magnetLink.split(separator: ":")[safe: 3],
let tempHash = firstSplit.split(separator: "&")[safe: 0]
{
magnetHash = String(tempHash)
} else {
return nil
}
@ -554,4 +828,27 @@ class ScrapingViewModel: ObservableObject {
return magnetLinkArray.joined()
}
@MainActor
func cleanApiCreds(api: SourceApi) {
let backgroundContext = PersistenceController.shared.backgroundContext
var clientIdReset = false
var clientSecretReset = false
if let clientId = api.clientId, !clientId.dynamic {
clientId.value = nil
clientIdReset = true
}
if let clientSecret = api.clientSecret, !clientSecret.dynamic {
clientSecret.value = nil
clientSecretReset = true
}
toastModel?.toastDescription =
"Could not fetch results, your \(clientIdReset ? "client ID" : "") \(clientIdReset && clientSecretReset ? "and" : "") \(clientSecretReset ? "token" : "") was automatically reset. Make sure all credentials are correct in the source's settings!"
PersistenceController.shared.save(backgroundContext)
}
}

125
Ferrite/ViewModels/SourceManager.swift
View file

@ -28,7 +28,10 @@ public class SourceManager: ObservableObject {
return
}
let (data, _) = try await URLSession.shared.data(for: URLRequest(url: url))
// Always get the up-to-date source list
let request = URLRequest(url: url, cachePolicy: .reloadIgnoringLocalCacheData)
let (data, _) = try await URLSession.shared.data(for: request)
var sourceResponse = try JSONDecoder().decode(SourceListJson.self, from: data)
for index in sourceResponse.sources.indices {
@ -45,6 +48,17 @@ public class SourceManager: ObservableObject {
}
}
// Fetches sources using the background context
public func fetchInstalledSources() -> [Source] {
let backgroundContext = PersistenceController.shared.backgroundContext
if let sources = try? backgroundContext.fetch(Source.fetchRequest()) {
return sources.compactMap { $0 }
} else {
return []
}
}
public func installSource(sourceJson: SourceJson, doUpsert: Bool = false) {
let backgroundContext = PersistenceController.shared.backgroundContext
@ -84,11 +98,16 @@ public class SourceManager: ObservableObject {
newSource.baseUrl = sourceJson.baseUrl
newSource.author = sourceJson.author ?? "Unknown"
newSource.listId = sourceJson.listId
newSource.trackers = sourceJson.trackers
if let sourceApiJson = sourceJson.api {
addSourceApi(newSource: newSource, apiJson: sourceApiJson)
}
if let jsonParserJson = sourceJson.jsonParser {
addJsonParser(newSource: newSource, jsonParserJson: jsonParserJson)
}
// Adds an RSS parser if present
if let rssParserJson = sourceJson.rssParser {
addRssParser(newSource: newSource, rssParserJson: rssParserJson)
@ -100,7 +119,9 @@ public class SourceManager: ObservableObject {
}
// Add an API condition as well
if newSource.rssParser != nil {
if newSource.jsonParser != nil {
newSource.preferredParser = Int16(SourcePreferredParser.siteApi.rawValue)
} else if newSource.rssParser != nil {
newSource.preferredParser = Int16(SourcePreferredParser.rss.rawValue)
} else {
newSource.preferredParser = Int16(SourcePreferredParser.scraping.rawValue)
@ -121,21 +142,98 @@ public class SourceManager: ObservableObject {
let backgroundContext = PersistenceController.shared.backgroundContext
let newSourceApi = SourceApi(context: backgroundContext)
newSourceApi.clientId = apiJson.clientId
newSourceApi.apiUrl = apiJson.apiUrl
if let clientId = apiJson.clientId {
newSourceApi.clientId = clientId
if let clientIdJson = apiJson.clientId {
let newClientId = SourceApiClientId(context: backgroundContext)
newClientId.query = clientIdJson.query
newClientId.urlString = clientIdJson.url
newClientId.dynamic = clientIdJson.dynamic ?? false
newClientId.value = clientIdJson.value
newClientId.responseType = clientIdJson.responseType?.rawValue ?? ApiCredentialResponseType.json.rawValue
newClientId.expiryLength = clientIdJson.expiryLength ?? 0
newClientId.timeStamp = Date()
newSourceApi.clientId = newClientId
}
newSourceApi.dynamicClientId = apiJson.dynamicClientId ?? false
if let clientSecretJson = apiJson.clientSecret {
let newClientSecret = SourceApiClientSecret(context: backgroundContext)
newClientSecret.query = clientSecretJson.query
newClientSecret.urlString = clientSecretJson.url
newClientSecret.dynamic = clientSecretJson.dynamic ?? false
newClientSecret.value = clientSecretJson.value
newClientSecret.responseType = clientSecretJson.responseType?.rawValue ?? ApiCredentialResponseType.json.rawValue
newClientSecret.expiryLength = clientSecretJson.expiryLength ?? 0
newClientSecret.timeStamp = Date()
if apiJson.usesSecret {
newSourceApi.clientSecret = ""
newSourceApi.clientSecret = newClientSecret
}
newSource.api = newSourceApi
}
func addJsonParser(newSource: Source, jsonParserJson: SourceJsonParserJson) {
let backgroundContext = PersistenceController.shared.backgroundContext
let newSourceJsonParser = SourceJsonParser(context: backgroundContext)
newSourceJsonParser.searchUrl = jsonParserJson.searchUrl
newSourceJsonParser.results = jsonParserJson.results
newSourceJsonParser.subResults = jsonParserJson.subResults
// Tune these complex queries to the final JSON parser format
if let magnetLinkJson = jsonParserJson.magnetLink {
let newSourceMagnetLink = SourceMagnetLink(context: backgroundContext)
newSourceMagnetLink.query = magnetLinkJson.query
newSourceMagnetLink.attribute = magnetLinkJson.attribute ?? "text"
newSourceMagnetLink.discriminator = magnetLinkJson.discriminator
newSourceJsonParser.magnetLink = newSourceMagnetLink
}
if let magnetHashJson = jsonParserJson.magnetHash {
let newSourceMagnetHash = SourceMagnetHash(context: backgroundContext)
newSourceMagnetHash.query = magnetHashJson.query
newSourceMagnetHash.attribute = magnetHashJson.attribute ?? "text"
newSourceMagnetHash.discriminator = magnetHashJson.discriminator
newSourceJsonParser.magnetHash = newSourceMagnetHash
}
if let titleJson = jsonParserJson.title {
let newSourceTitle = SourceTitle(context: backgroundContext)
newSourceTitle.query = titleJson.query
newSourceTitle.attribute = titleJson.attribute ?? "text"
newSourceTitle.discriminator = titleJson.discriminator
newSourceJsonParser.title = newSourceTitle
}
if let sizeJson = jsonParserJson.size {
let newSourceSize = SourceSize(context: backgroundContext)
newSourceSize.query = sizeJson.query
newSourceSize.attribute = sizeJson.attribute ?? "text"
newSourceSize.discriminator = sizeJson.discriminator
newSourceJsonParser.size = newSourceSize
}
if let seedLeechJson = jsonParserJson.sl {
let newSourceSeedLeech = SourceSeedLeech(context: backgroundContext)
newSourceSeedLeech.seeders = seedLeechJson.seeders
newSourceSeedLeech.leechers = seedLeechJson.leechers
newSourceSeedLeech.combined = seedLeechJson.combined
newSourceSeedLeech.attribute = seedLeechJson.attribute ?? "text"
newSourceSeedLeech.discriminator = seedLeechJson.discriminator
newSourceSeedLeech.seederRegex = seedLeechJson.seederRegex
newSourceSeedLeech.leecherRegex = seedLeechJson.leecherRegex
newSourceJsonParser.seedLeech = newSourceSeedLeech
}
newSource.jsonParser = newSourceJsonParser
}
func addRssParser(newSource: Source, rssParserJson: SourceRssParserJson) {
let backgroundContext = PersistenceController.shared.backgroundContext
@ -143,13 +241,12 @@ public class SourceManager: ObservableObject {
newSourceRssParser.rssUrl = rssParserJson.rssUrl
newSourceRssParser.searchUrl = rssParserJson.searchUrl
newSourceRssParser.items = rssParserJson.items
newSourceRssParser.trackers = rssParserJson.trackers
if let magnetLinkJson = rssParserJson.magnetLink {
let newSourceMagnetLink = SourceMagnetLink(context: backgroundContext)
newSourceMagnetLink.query = magnetLinkJson.query
newSourceMagnetLink.attribute = magnetLinkJson.attribute ?? "text"
newSourceMagnetLink.lookupAttribute = magnetLinkJson.lookupAttribute
newSourceMagnetLink.discriminator = magnetLinkJson.discriminator
newSourceRssParser.magnetLink = newSourceMagnetLink
}
@ -158,7 +255,7 @@ public class SourceManager: ObservableObject {
let newSourceMagnetHash = SourceMagnetHash(context: backgroundContext)
newSourceMagnetHash.query = magnetHashJson.query
newSourceMagnetHash.attribute = magnetHashJson.attribute ?? "text"
newSourceMagnetHash.lookupAttribute = magnetHashJson.lookupAttribute
newSourceMagnetHash.discriminator = magnetHashJson.discriminator
newSourceRssParser.magnetHash = newSourceMagnetHash
}
@ -167,7 +264,7 @@ public class SourceManager: ObservableObject {
let newSourceTitle = SourceTitle(context: backgroundContext)
newSourceTitle.query = titleJson.query
newSourceTitle.attribute = titleJson.attribute ?? "text"
newSourceTitle.lookupAttribute = newSourceTitle.lookupAttribute
newSourceTitle.discriminator = titleJson.discriminator
newSourceRssParser.title = newSourceTitle
}
@ -176,7 +273,7 @@ public class SourceManager: ObservableObject {
let newSourceSize = SourceSize(context: backgroundContext)
newSourceSize.query = sizeJson.query
newSourceSize.attribute = sizeJson.attribute ?? "text"
newSourceSize.lookupAttribute = sizeJson.lookupAttribute
newSourceSize.discriminator = sizeJson.discriminator
newSourceRssParser.size = newSourceSize
}
@ -187,7 +284,7 @@ public class SourceManager: ObservableObject {
newSourceSeedLeech.leechers = seedLeechJson.leechers
newSourceSeedLeech.combined = seedLeechJson.combined
newSourceSeedLeech.attribute = seedLeechJson.attribute ?? "text"
newSourceSeedLeech.lookupAttribute = seedLeechJson.lookupAttribute
newSourceSeedLeech.discriminator = seedLeechJson.discriminator
newSourceSeedLeech.seederRegex = seedLeechJson.seederRegex
newSourceSeedLeech.leecherRegex = seedLeechJson.leecherRegex

9
Ferrite/Views/ContentView.swift
View file

@ -12,6 +12,7 @@ struct ContentView: View {
@EnvironmentObject var scrapingModel: ScrapingViewModel
@EnvironmentObject var debridManager: DebridManager
@EnvironmentObject var navModel: NavigationViewModel
@EnvironmentObject var sourceManager: SourceManager
@AppStorage("RealDebrid.Enabled") var realDebridEnabled = false
@ -98,13 +99,17 @@ struct ContentView: View {
}
.navigationTitle("Search")
.navigationSearchBar {
SearchBar("Search", text: $scrapingModel.searchText, isEditing: $navModel.isEditingSearch,
SearchBar("Search",
text: $scrapingModel.searchText,
isEditing: $navModel.isEditingSearch,
onCommit: {
scrapingModel.searchResults = []
scrapingModel.runningSearchTask = Task {
navModel.isSearching = true
navModel.showSearchProgress = true
await scrapingModel.scanSources(sources: sources.compactMap { $0 })
let sources = sourceManager.fetchInstalledSources()
await scrapingModel.scanSources(sources: sources)
if realDebridEnabled, !scrapingModel.searchResults.isEmpty {
await debridManager.populateDebridHashes(scrapingModel.searchResults)

5
Ferrite/Views/MagnetChoiceView.swift
View file

@ -74,7 +74,10 @@ struct MagnetChoiceView: View {
}
ListRowButtonView("Share magnet", systemImage: "square.and.arrow.up.fill") {
if let result = scrapingModel.selectedSearchResult, let url = URL(string: result.magnetLink) {
if let result = scrapingModel.selectedSearchResult,
let magnetLink = result.magnetLink,
let url = URL(string: magnetLink)
{
activityItems = [url]
navModel.showActivityView.toggle()
}

4
Ferrite/Views/SearchResultRDView.swift
View file

@ -28,7 +28,9 @@ struct SearchResultRDView: View {
Text("L: \(leechers)")
}
Text(result.size)
if let size = result.size {
Text(size)
}
if realDebridEnabled {
Text("RD")

2
Ferrite/Views/SearchResultsView.swift
View file

@ -39,7 +39,7 @@ struct SearchResultsView: View {
navModel.runMagnetAction(action: nil, searchResult: result)
}
} label: {
Text(result.title)
Text(result.title ?? "No title")
.font(.callout)
.fixedSize(horizontal: false, vertical: true)
}

29
Ferrite/Views/SourceViews/SourceSettingsView.swift
View file

@ -46,7 +46,9 @@ struct SourceSettingsView: View {
SourceSettingsBaseUrlView(selectedSource: selectedSource)
}
if let sourceApi = selectedSource.api {
if let sourceApi = selectedSource.api,
sourceApi.clientId?.dynamic ?? false || sourceApi.clientSecret?.dynamic ?? false
{
SourceSettingsApiView(selectedSourceApi: sourceApi)
}
@ -110,27 +112,29 @@ struct SourceSettingsApiView: View {
header: Text("API credentials"),
footer: Text("Grab the required API credentials from the website. A client secret can be an API token.")
) {
if selectedSourceApi.dynamicClientId {
if let clientId = selectedSourceApi.clientId, clientId.dynamic {
TextField("Client ID", text: $tempClientId, onEditingChanged: { isFocused in
if !isFocused {
selectedSourceApi.clientId = tempClientId
clientId.value = tempClientId
clientId.timeStamp = Date()
}
})
.autocapitalization(.none)
.onAppear {
tempClientId = selectedSourceApi.clientId ?? ""
tempClientId = clientId.value ?? ""
}
}
if selectedSourceApi.clientSecret != nil {
if let clientSecret = selectedSourceApi.clientSecret, clientSecret.dynamic {
TextField("Token", text: $tempClientSecret, onEditingChanged: { isFocused in
if !isFocused {
selectedSourceApi.clientSecret = tempClientSecret
clientSecret.value = tempClientSecret
clientSecret.timeStamp = Date()
}
})
.autocapitalization(.none)
.onAppear {
tempClientSecret = selectedSourceApi.clientSecret ?? ""
tempClientSecret = clientSecret.value ?? ""
}
}
}
@ -144,15 +148,20 @@ struct SourceSettingsMethodView: View {
var body: some View {
Picker("Fetch method", selection: $selectedTempParser) {
if selectedSource.htmlParser != nil {
Text("Web scraping")
.tag(SourcePreferredParser.scraping)
if selectedSource.jsonParser != nil {
Text("Website API")
.tag(SourcePreferredParser.siteApi)
}
if selectedSource.rssParser != nil {
Text("RSS")
.tag(SourcePreferredParser.rss)
}
if selectedSource.htmlParser != nil {
Text("Web scraping")
.tag(SourcePreferredParser.scraping)
}
}
.pickerStyle(.inline)
.onAppear {