diff --git a/Sora/Utlis & Misc/Extensions/JavaScriptCore+Extensions.swift b/Sora/Utlis & Misc/Extensions/JavaScriptCore+Extensions.swift index 9fb8f8b..581e68a 100644 --- a/Sora/Utlis & Misc/Extensions/JavaScriptCore+Extensions.swift +++ b/Sora/Utlis & Misc/Extensions/JavaScriptCore+Extensions.swift @@ -385,6 +385,7 @@ extension JSContext { setupConsoleLogging() setupNativeFetch() setupNetworkFetch() + setupNetworkFetchSimple() setupFetchV2() setupBase64Functions() setupScrapingUtilities() diff --git a/Sora/Utlis & Misc/JSLoader/JSController-NetworkFetch.swift b/Sora/Utlis & Misc/JSLoader/JSController-NetworkFetch.swift index 413b0b9..bc2bfe9 100644 --- a/Sora/Utlis & Misc/JSLoader/JSController-NetworkFetch.swift +++ b/Sora/Utlis & Misc/JSLoader/JSController-NetworkFetch.swift @@ -13,26 +13,32 @@ struct NetworkFetchOptions { let headers: [String: String] let cutoff: String? let returnHTML: Bool + let returnCookies: Bool let clickSelectors: [String] let waitForSelectors: [String] let maxWaitTime: Int + let htmlContent: String? init( timeoutSeconds: Int = 10, headers: [String: String] = [:], cutoff: String? = nil, returnHTML: Bool = false, + returnCookies: Bool = true, clickSelectors: [String] = [], waitForSelectors: [String] = [], - maxWaitTime: Int = 5 + maxWaitTime: Int = 5, + htmlContent: String? = nil ) { self.timeoutSeconds = timeoutSeconds self.headers = headers self.cutoff = cutoff self.returnHTML = returnHTML + self.returnCookies = returnCookies self.clickSelectors = clickSelectors self.waitForSelectors = waitForSelectors self.maxWaitTime = maxWaitTime + self.htmlContent = htmlContent } } @@ -47,18 +53,22 @@ extension JSContext { let headers = optionsDict["headers"] as? [String: String] ?? [:] let cutoff = optionsDict["cutoff"] as? String let returnHTML = optionsDict["returnHTML"] as? Bool ?? false + let returnCookies = optionsDict["returnCookies"] as? Bool ?? true let clickSelectors = optionsDict["clickSelectors"] as? [String] ?? [] let waitForSelectors = optionsDict["waitForSelectors"] as? [String] ?? [] let maxWaitTime = optionsDict["maxWaitTime"] as? Int ?? 5 + let htmlContent = optionsDict["htmlContent"] as? String options = NetworkFetchOptions( timeoutSeconds: timeoutSeconds, headers: headers, cutoff: cutoff, returnHTML: returnHTML, + returnCookies: returnCookies, clickSelectors: clickSelectors, waitForSelectors: waitForSelectors, - maxWaitTime: maxWaitTime + maxWaitTime: maxWaitTime, + htmlContent: htmlContent ) } @@ -87,9 +97,11 @@ extension JSContext { headers: options.headers || {}, cutoff: options.cutoff || null, returnHTML: options.returnHTML || false, + returnCookies: options.returnCookies !== undefined ? options.returnCookies : true, clickSelectors: options.clickSelectors || [], waitForSelectors: options.waitForSelectors || [], - maxWaitTime: options.maxWaitTime || 5 + maxWaitTime: options.maxWaitTime || 5, + htmlContent: options.htmlContent || null }; return new Promise(function(resolve, reject) { @@ -98,12 +110,14 @@ extension JSContext { url: result.originalUrl, requests: result.requests, html: result.html || null, + cookies: result.cookies || null, success: result.success, error: result.error || null, totalRequests: result.requests.length, cutoffTriggered: result.cutoffTriggered || false, cutoffUrl: result.cutoffUrl || null, htmlCaptured: result.htmlCaptured || false, + cookiesCaptured: result.cookiesCaptured || false, elementsClicked: result.elementsClicked || [], waitResults: result.waitResults || {} }); @@ -114,14 +128,16 @@ extension JSContext { function networkFetchWithHTML(url, timeoutSeconds = 10) { return networkFetch(url, { timeoutSeconds: timeoutSeconds, - returnHTML: true + returnHTML: true, + returnCookies: true }); } function networkFetchWithCutoff(url, cutoff, timeoutSeconds = 10) { return networkFetch(url, { timeoutSeconds: timeoutSeconds, - cutoff: cutoff + cutoff: cutoff, + returnCookies: true }); } @@ -131,6 +147,7 @@ extension JSContext { headers: options.headers || {}, cutoff: options.cutoff || null, returnHTML: options.returnHTML || false, + returnCookies: options.returnCookies !== undefined ? options.returnCookies : true, clickSelectors: Array.isArray(clickSelectors) ? clickSelectors : [clickSelectors], waitForSelectors: options.waitForSelectors || [], maxWaitTime: options.maxWaitTime || 5 @@ -143,140 +160,710 @@ extension JSContext { headers: options.headers || {}, cutoff: options.cutoff || null, returnHTML: options.returnHTML || false, + returnCookies: options.returnCookies !== undefined ? options.returnCookies : true, clickSelectors: Array.isArray(clickSelectors) ? clickSelectors : [clickSelectors], waitForSelectors: Array.isArray(waitForSelectors) ? waitForSelectors : [waitForSelectors], maxWaitTime: options.maxWaitTime || 5 }); } + + function networkFetchFromHTML(htmlContent, options = {}) { + return networkFetch('', { + timeoutSeconds: options.timeoutSeconds || 10, + headers: options.headers || {}, + cutoff: options.cutoff || null, + returnHTML: options.returnHTML || false, + returnCookies: options.returnCookies !== undefined ? options.returnCookies : true, + clickSelectors: options.clickSelectors || [], + waitForSelectors: options.waitForSelectors || [], + maxWaitTime: options.maxWaitTime || 5, + htmlContent: htmlContent + }); + } """ self.evaluateScript(networkFetchDefinition) } + + func setupNetworkFetchSimple() { + let networkFetchSimpleNativeFunction: @convention(block) (String, JSValue?, JSValue, JSValue) -> Void = { urlString, optionsValue, resolve, reject in + DispatchQueue.main.async { + var timeoutSeconds = 5 + var htmlContent: String? = nil + var headers: [String: String] = [:] + if let optionsDict = optionsValue?.toDictionary() { + timeoutSeconds = optionsDict["timeoutSeconds"] as? Int ?? 5 + htmlContent = optionsDict["htmlContent"] as? String + headers = optionsDict["headers"] as? [String: String] ?? [:] + } + NetworkFetchSimpleManager.shared.performNetworkFetch( + urlString: urlString, + timeoutSeconds: timeoutSeconds, + htmlContent: htmlContent, + headers: headers, + resolve: resolve, + reject: reject + ) + } + } + self.setObject(networkFetchSimpleNativeFunction, forKeyedSubscript: "networkFetchSimpleNative" as NSString) + let networkFetchSimpleDefinition = """ + function networkFetchSimple(url, options = {}) { + if (typeof options === 'number') { + const timeoutSeconds = options; + options = { timeoutSeconds }; + } + const finalOptions = { + timeoutSeconds: options.timeoutSeconds || 5, + htmlContent: options.htmlContent || null, + headers: options.headers || {} + }; + return new Promise(function(resolve, reject) { + networkFetchSimpleNative(url, finalOptions, function(result) { + resolve({ + url: result.originalUrl, + requests: result.requests, + success: result.success, + error: result.error || null, + totalRequests: result.requests.length + }); + }, reject); + }); + } + function networkFetchSimpleFromHTML(htmlContent, options = {}) { + return networkFetchSimple('', { + timeoutSeconds: options.timeoutSeconds || 5, + htmlContent: htmlContent, + headers: options.headers || {} + }); + } + """ + self.evaluateScript(networkFetchSimpleDefinition) + } } -class NetworkFetchManager: NSObject, ObservableObject { - static let shared = NetworkFetchManager() +class NetworkFetchSimpleManager: NSObject, ObservableObject { + static let shared = NetworkFetchSimpleManager() - private var activeMonitors: [String: NetworkFetchMonitor] = [:] + private var activeMonitors: [String: NetworkFetchSimpleMonitor] = [:] private override init() { super.init() } - func performNetworkFetch(urlString: String, options: NetworkFetchOptions, resolve: JSValue, reject: JSValue) { - Logger.shared.log("NetworkFetchManager: Starting fetch for \(urlString) with options: returnHTML=\(options.returnHTML), clicks=\(options.clickSelectors), waitFor=\(options.waitForSelectors)", type: "Debug") - + func performNetworkFetch(urlString: String, timeoutSeconds: Int, htmlContent: String? = nil, headers: [String: String] = [:], resolve: JSValue, reject: JSValue) { let monitorId = UUID().uuidString - let monitor = NetworkFetchMonitor() + let monitor = NetworkFetchSimpleMonitor() activeMonitors[monitorId] = monitor - monitor.startMonitoring( urlString: urlString, - options: options + timeoutSeconds: timeoutSeconds, + htmlContent: htmlContent, + headers: headers ) { [weak self] result in - Logger.shared.log("NetworkFetchManager: Fetch completed for \(urlString)", type: "Debug") - self?.activeMonitors.removeValue(forKey: monitorId) - DispatchQueue.main.async { if !resolve.isUndefined { - Logger.shared.log("NetworkFetchManager: Calling resolve with result", type: "Debug") resolve.call(withArguments: [result]) - } else { - Logger.shared.log("NetworkFetchManager: Resolve callback is undefined!", type: "Error") } } } } } -class NetworkFetchMonitor: NSObject, ObservableObject { +class NetworkFetchSimpleMonitor: NSObject, ObservableObject { private var webView: WKWebView? private var completionHandler: (([String: Any]) -> Void)? private var timer: Timer? - private var options: NetworkFetchOptions? - private var elementsClicked: [String] = [] - private var waitResults: [String: Bool] = [:] @Published private(set) var networkRequests: [String] = [] - @Published private(set) var statusMessage = "Initializing..." - @Published private(set) var cutoffTriggered = false - @Published private(set) var cutoffUrl: String? = nil - @Published private(set) var htmlContent: String? = nil - @Published private(set) var htmlCaptured = false - func startMonitoring(urlString: String, options: NetworkFetchOptions, completion: @escaping ([String: Any]) -> Void) { - self.options = options + private var originalUrlString: String = "" + + func startMonitoring(urlString: String, timeoutSeconds: Int, htmlContent: String? = nil, headers: [String: String] = [:], completion: @escaping ([String: Any]) -> Void) { + originalUrlString = urlString completionHandler = completion networkRequests.removeAll() - cutoffTriggered = false - cutoffUrl = nil - htmlContent = nil - htmlCaptured = false - elementsClicked.removeAll() - waitResults.removeAll() - - var statusParts = ["Loading URL for \(options.timeoutSeconds) seconds"] - if !options.waitForSelectors.isEmpty { - statusParts.append("waiting for elements") - } - if !options.clickSelectors.isEmpty { - statusParts.append("will click elements") - } - if options.returnHTML { - statusParts.append("will capture HTML") - } - statusMessage = statusParts.joined(separator: ", ") + "..." - - guard let url = URL(string: urlString) else { - completion([ - "originalUrl": urlString, - "requests": [], - "html": NSNull(), - "success": false, - "error": "Invalid URL format", - "htmlCaptured": false, - "elementsClicked": [], - "waitResults": [:] - ]) - return - } - - setupWebView() - loadURL(url: url, headers: options.headers) - - timer = Timer.scheduledTimer(withTimeInterval: TimeInterval(options.timeoutSeconds), repeats: false) { [weak self] _ in - if options.returnHTML { - self?.captureHTMLThenComplete() - } else { - self?.stopMonitoring(reason: "timeout") + if let htmlContent = htmlContent, !htmlContent.isEmpty { + setupWebView() + loadHTMLContent(htmlContent) + } else { + guard let url = URL(string: urlString) else { + completion([ + "originalUrl": urlString, + "requests": [], + "success": false, + "error": "Invalid URL format" + ]) + return } + setupWebView() + loadURL(url: url, headers: headers) + } + timer = Timer.scheduledTimer(withTimeInterval: TimeInterval(timeoutSeconds), repeats: false) { [weak self] _ in + self?.stopMonitoring() } - - Logger.shared.log("NetworkFetch started for: \(urlString) (timeout: \(options.timeoutSeconds)s, returnHTML: \(options.returnHTML), clicks: \(options.clickSelectors), waitFor: \(options.waitForSelectors))", type: "Debug") } - private func captureHTMLThenComplete() { - guard let webView = webView, let options = options, options.returnHTML else { - stopMonitoring(reason: "timeout") - return - } + private func loadHTMLContent(_ htmlContent: String) { + guard let webView = webView else { return } - statusMessage = "Capturing HTML content before timeout..." - Logger.shared.log("NetworkFetch: Capturing HTML at timeout", type: "Debug") + addRequest("data:text/html;charset=utf-8,") - webView.evaluateJavaScript("document.documentElement.outerHTML") { [weak self] result, error in - DispatchQueue.main.async { - if let html = result as? String, error == nil { - self?.htmlContent = html - self?.htmlCaptured = true - Logger.shared.log("NetworkFetch: HTML captured successfully (\(html.count) characters)", type: "Debug") - } else { - Logger.shared.log("NetworkFetch: Failed to capture HTML: \(error?.localizedDescription ?? "Unknown error")", type: "Error") - } - - self?.stopMonitoring(reason: "timeout_with_html") - } + webView.loadHTMLString(htmlContent, baseURL: nil) + + DispatchQueue.main.asyncAfter(deadline: .now() + 2.0) { + self.simulateUserInteraction() + } + } + + private func setupWebView() { + let config = WKWebViewConfiguration() + config.allowsInlineMediaPlayback = true + config.mediaTypesRequiringUserActionForPlayback = [] + + let jsCode = """ + (function() { + Object.defineProperty(navigator, 'webdriver', { get: () => undefined }); + Object.defineProperty(navigator, 'plugins', { get: () => [1, 2, 3, 4, 5] }); + Object.defineProperty(navigator, 'languages', { get: () => ['en-US', 'en'] }); + delete window.navigator.__proto__.webdriver; + + window.chrome = { runtime: {} }; + Object.defineProperty(navigator, 'permissions', { get: () => undefined }); + + const originalFetch = window.fetch; + const originalXHROpen = XMLHttpRequest.prototype.open; + const originalXHRSend = XMLHttpRequest.prototype.send; + + window.fetch = function() { + const url = arguments[0]; + const options = arguments[1] || {}; + + try { + const fullUrl = new URL(url, window.location.href).href; + window.webkit.messageHandlers.networkLogger.postMessage({ + type: 'fetch', + url: fullUrl + }); + } catch(e) { + window.webkit.messageHandlers.networkLogger.postMessage({ + type: 'fetch', + url: url.toString() + }); + } + return originalFetch.apply(this, arguments); + }; + + XMLHttpRequest.prototype.open = function() { + const method = arguments[0]; + const url = arguments[1]; + + try { + this._url = new URL(url, window.location.href).href; + } catch(e) { + this._url = url; + } + + window.webkit.messageHandlers.networkLogger.postMessage({ + type: 'xhr-open', + url: this._url + }); + + const self = this; + const originalOnReadyStateChange = this.onreadystatechange; + + this.onreadystatechange = function() { + if (this.readyState === 4) { + if (this.responseURL) { + window.webkit.messageHandlers.networkLogger.postMessage({ + type: 'xhr-response', + url: this.responseURL + }); + } + + try { + const responseText = this.responseText; + if (responseText) { + const urlRegex = /(https?:\\/\\/[^\\s"'<>]+\\.(m3u8|ts|mp4|webm|mkv))/gi; + const matches = responseText.match(urlRegex); + if (matches) { + matches.forEach(function(match) { + window.webkit.messageHandlers.networkLogger.postMessage({ + type: 'response-content', + url: match + }); + }); + } + } + } catch(e) { + } + } + + if (originalOnReadyStateChange) { + originalOnReadyStateChange.apply(this, arguments); + } + }; + + return originalXHROpen.apply(this, arguments); + }; + + XMLHttpRequest.prototype.send = function() { + if (this._url) { + window.webkit.messageHandlers.networkLogger.postMessage({ + type: 'xhr-send', + url: this._url + }); + } + return originalXHRSend.apply(this, arguments); + }; + + const originalWebSocket = window.WebSocket; + window.WebSocket = function(url, protocols) { + window.webkit.messageHandlers.networkLogger.postMessage({ + type: 'websocket', + url: url + }); + return new originalWebSocket(url, protocols); + }; + + const hookUrlProperties = function(obj, properties) { + properties.forEach(function(prop) { + if (obj && obj.prototype) { + const descriptor = Object.getOwnPropertyDescriptor(obj.prototype, prop) || {}; + const originalSetter = descriptor.set; + + if (originalSetter) { + Object.defineProperty(obj.prototype, prop, { + set: function(value) { + if (typeof value === 'string' && (value.includes('http') || value.includes('.m3u8') || value.includes('.ts'))) { + window.webkit.messageHandlers.networkLogger.postMessage({ + type: 'property-set', + url: value + }); + } + return originalSetter.call(this, value); + }, + get: descriptor.get, + configurable: true + }); + } + } + }); + }; + + hookUrlProperties(HTMLVideoElement, ['src']); + hookUrlProperties(HTMLSourceElement, ['src']); + hookUrlProperties(HTMLScriptElement, ['src']); + hookUrlProperties(HTMLImageElement, ['src']); + + let jwHookAttempts = 0; + const aggressiveJWHook = function() { + jwHookAttempts++; + + if (window.jwplayer) { + const originalJWPlayer = window.jwplayer; + window.jwplayer = function(id) { + const player = originalJWPlayer.apply(this, arguments); + + if (player && player.setup) { + const originalSetup = player.setup; + player.setup = function(config) { + const extractUrls = function(obj, path = '') { + if (!obj) return; + + if (typeof obj === 'string' && (obj.includes('http') || obj.includes('.m3u8') || obj.includes('.ts'))) { + window.webkit.messageHandlers.networkLogger.postMessage({ + type: 'jwplayer-config', + url: obj + }); + } else if (typeof obj === 'object' && obj !== null) { + Object.keys(obj).forEach(function(key) { + extractUrls(obj[key], path + '.' + key); + }); + } + }; + + extractUrls(config); + return originalSetup.call(this, config); + }; + } + + return player; + }; + + Object.keys(originalJWPlayer).forEach(function(key) { + window.jwplayer[key] = originalJWPlayer[key]; + }); + } + + if (jwHookAttempts < 20) { + setTimeout(aggressiveJWHook, 200); + } + }; + + aggressiveJWHook(); + + const nuclearScan = function() { + Object.keys(window).forEach(function(key) { + try { + const value = window[key]; + if (typeof value === 'string' && (value.includes('.m3u8') || value.includes('.ts') || (value.includes('http') && value.includes('.')))) { + window.webkit.messageHandlers.networkLogger.postMessage({ + type: 'global-variable', + url: value + }); + } + } catch(e) { + } + }); + + document.querySelectorAll('script').forEach(function(script) { + if (script.textContent) { + const urlRegex = /(https?:\\/\\/[^\\s"'<>]+\\.(m3u8|ts|mp4))/gi; + const matches = script.textContent.match(urlRegex); + if (matches) { + matches.forEach(function(match) { + window.webkit.messageHandlers.networkLogger.postMessage({ + type: 'script-content', + url: match + }); + }); + } + } + }); + + const clickableSelectors = [ + 'button', '.play', '.play-button', '[data-play]', '.video-play', + '.jwplayer', '.player', '[id*="player"]', '[class*="play"]', + 'div[onclick]', 'span[onclick]', 'a[onclick]' + ]; + + clickableSelectors.forEach(function(selector) { + document.querySelectorAll(selector).forEach(function(el) { + try { + el.click(); + } catch(e) { + } + }); + }); + }; + + setTimeout(nuclearScan, 500); + setTimeout(nuclearScan, 1500); + setTimeout(nuclearScan, 3000); + })(); + """ + + let userScript = WKUserScript(source: jsCode, injectionTime: .atDocumentStart, forMainFrameOnly: false) + config.userContentController.addUserScript(userScript) + config.userContentController.add(self, name: "networkLogger") + + webView = WKWebView(frame: CGRect(x: 0, y: 0, width: 1920, height: 1080), configuration: config) + webView?.navigationDelegate = self + + webView?.customUserAgent = URLSession.randomUserAgent + } + + private func loadURL(url: URL, headers: [String: String] = [:]) { + guard let webView = webView else { return } + addRequest(url.absoluteString) + var request = URLRequest(url: url) + request.setValue(URLSession.randomUserAgent, forHTTPHeaderField: "User-Agent") + request.setValue("text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8", forHTTPHeaderField: "Accept") + request.setValue("en-US,en;q=0.5", forHTTPHeaderField: "Accept-Language") + request.setValue("gzip, deflate, br", forHTTPHeaderField: "Accept-Encoding") + request.setValue("no-cache", forHTTPHeaderField: "Cache-Control") + request.setValue("upgrade-insecure-requests", forHTTPHeaderField: "Upgrade-Insecure-Requests") + request.setValue("same-origin", forHTTPHeaderField: "Sec-Fetch-Site") + request.setValue("navigate", forHTTPHeaderField: "Sec-Fetch-Mode") + for (key, value) in headers { + request.setValue(value, forHTTPHeaderField: key) + } + let randomReferers = [ + "https://www.google.com/", + "https://www.youtube.com/", + "https://twitter.com/", + "https://www.reddit.com/", + "https://www.facebook.com/" + ] + let randomReferer = randomReferers.randomElement() ?? "https://www.google.com/" + if request.value(forHTTPHeaderField: "Referer") == nil { + request.setValue(randomReferer, forHTTPHeaderField: "Referer") + } + webView.load(request) + DispatchQueue.main.asyncAfter(deadline: .now() + 2.0) { + self.simulateUserInteraction() + } + } + + private func simulateUserInteraction() { + guard let webView = webView else { return } + + let jsInteraction = """ + setTimeout(function() { + const playButtons = document.querySelectorAll('button, div, span, a').filter(function(el) { + const text = el.textContent || el.innerText || ''; + const classes = el.className || ''; + return text.toLowerCase().includes('play') || + classes.toLowerCase().includes('play') || + el.getAttribute('aria-label')?.toLowerCase().includes('play'); + }); + playButtons.forEach(function(btn, index) { + setTimeout(function() { + btn.click(); + }, index * 200); + }); + window.scrollTo(0, document.body.scrollHeight / 2); + setTimeout(function() { + window.scrollTo(0, 0); + }, 500); + document.querySelectorAll('video').forEach(function(video) { + if (video.play && typeof video.play === 'function') { + video.play().catch(function(e) { + }); + } + }); + if (window.jwplayer) { + try { + const players = window.jwplayer().getInstances?.() || []; + players.forEach(function(player) { + if (player.play) { + player.play(); + } + }); + } catch(e) {} + } + if (window.videojs) { + try { + window.videojs.getAllPlayers?.().forEach(function(player) { + if (player.play) { + player.play(); + } + }); + } catch(e) {} + } + }, 1000); + """ + webView.evaluateJavaScript(jsInteraction, completionHandler: nil) + } + + private func stopMonitoring() { + timer?.invalidate() + timer = nil + + webView?.stopLoading() + webView?.configuration.userContentController.removeScriptMessageHandler(forName: "networkLogger") + + let originalUrl = networkRequests.first == "data:text/html;charset=utf-8," ? + "data:text/html;charset=utf-8," : + (webView?.url?.absoluteString ?? originalUrlString) + + let result: [String: Any] = [ + "originalUrl": originalUrl, + "requests": networkRequests, + "success": true + ] + + webView = nil + + completionHandler?(result) + completionHandler = nil + } + + private func addRequest(_ urlString: String) { + DispatchQueue.main.async { + if !self.networkRequests.contains(urlString) { + self.networkRequests.append(urlString) + } + } + } +} + +extension NetworkFetchSimpleMonitor: WKNavigationDelegate { + func webView(_ webView: WKWebView, didFinish navigation: WKNavigation!) { + } + + func webView(_ webView: WKWebView, didFail navigation: WKNavigation!, withError error: Error) { + } + + func webView(_ webView: WKWebView, decidePolicyFor navigationAction: WKNavigationAction, decisionHandler: @escaping (WKNavigationActionPolicy) -> Void) { + if let url = navigationAction.request.url { + addRequest(url.absoluteString) + } + decisionHandler(.allow) + } +} + +extension NetworkFetchSimpleMonitor: WKScriptMessageHandler { + func userContentController(_ userContentController: WKUserContentController, didReceive message: WKScriptMessage) { + if message.name == "networkLogger" { + if let messageBody = message.body as? [String: Any], + let url = messageBody["url"] as? String { + addRequest(url) + } + } + } +} + +class NetworkFetchManager: NSObject, ObservableObject { + static let shared = NetworkFetchManager() + + private var activeMonitors: [String: NetworkFetchMonitor] = [:] + + private override init() { + super.init() + } + + func performNetworkFetch(urlString: String, options: NetworkFetchOptions, resolve: JSValue, reject: JSValue) { + let monitorId = UUID().uuidString + let monitor = NetworkFetchMonitor() + activeMonitors[monitorId] = monitor + + monitor.startMonitoring( + urlString: urlString, + options: options + ) { [weak self] result in + self?.activeMonitors.removeValue(forKey: monitorId) + + DispatchQueue.main.async { + if !resolve.isUndefined { + resolve.call(withArguments: [result]) + } + } + } + } +} + +class NetworkFetchMonitor: NSObject, ObservableObject { + private var webView: WKWebView? + private var completionHandler: (([String: Any]) -> Void)? + private var timer: Timer? + private var options: NetworkFetchOptions? + private var elementsClicked: [String] = [] + private var waitResults: [String: Bool] = [:] + private var cookies: [String: String] = [:] + + @Published private(set) var networkRequests: [String] = [] + @Published private(set) var cutoffTriggered = false + @Published private(set) var cutoffUrl: String? = nil + @Published private(set) var htmlContent: String? = nil + @Published private(set) var htmlCaptured = false + @Published private(set) var cookiesCaptured = false + + func startMonitoring(urlString: String, options: NetworkFetchOptions, completion: @escaping ([String: Any]) -> Void) { + self.options = options + completionHandler = completion + networkRequests.removeAll() + cutoffTriggered = false + cutoffUrl = nil + htmlContent = nil + htmlCaptured = false + cookiesCaptured = false + elementsClicked.removeAll() + waitResults.removeAll() + cookies.removeAll() + + + if let htmlContent = options.htmlContent, !htmlContent.isEmpty { + setupWebView() + loadHTMLContent(htmlContent) + } else { + guard let url = URL(string: urlString) else { + completion([ + "originalUrl": urlString, + "requests": [], + "html": NSNull(), + "cookies": NSNull(), + "success": false, + "error": "Invalid URL format", + "htmlCaptured": false, + "cookiesCaptured": false, + "elementsClicked": [], + "waitResults": [:] + ]) + return + } + + setupWebView() + loadURL(url: url, headers: options.headers) + } + + timer = Timer.scheduledTimer(withTimeInterval: TimeInterval(options.timeoutSeconds), repeats: false) { [weak self] _ in + if options.returnHTML || options.returnCookies { + self?.captureDataThenComplete() + } else { + self?.stopMonitoring(reason: "timeout") + } + } + } + + private func captureDataThenComplete() { + guard let webView = webView, let options = options else { + stopMonitoring(reason: "timeout") + return + } + + let shouldCaptureHTML = options.returnHTML + let shouldCaptureCookies = options.returnCookies + + var completedTasks = 0 + let totalTasks = (shouldCaptureHTML ? 1 : 0) + (shouldCaptureCookies ? 1 : 0) + + if totalTasks == 0 { + stopMonitoring(reason: "timeout") + return + } + + let checkCompletion = { + completedTasks += 1 + if completedTasks >= totalTasks { + self.stopMonitoring(reason: "timeout_with_data") + } + } + + + if shouldCaptureHTML { + webView.evaluateJavaScript("document.documentElement.outerHTML") { [weak self] result, error in + DispatchQueue.main.async { + if let html = result as? String, error == nil { + self?.htmlContent = html + self?.htmlCaptured = true + } + checkCompletion() + } + } + } + + if shouldCaptureCookies { + captureCookies { [weak self] in + DispatchQueue.main.async { + checkCompletion() + } + } + } + } + + private func captureCookies(completion: @escaping () -> Void) { + guard let webView = webView else { + completion() + return + } + + let cookieStore = webView.configuration.websiteDataStore.httpCookieStore + + cookieStore.getAllCookies { [weak self] cookies in + DispatchQueue.main.async { + var cookieDict: [String: String] = [:] + for cookie in cookies { + cookieDict[cookie.name] = cookie.value + } + + self?.cookies = cookieDict + self?.cookiesCaptured = !cookieDict.isEmpty + + completion() + } } } @@ -287,8 +874,6 @@ class NetworkFetchMonitor: NSObject, ObservableObject { let jsCode = """ (function() { - console.log('Advanced network interceptor loaded'); - Object.defineProperty(navigator, 'webdriver', { get: () => undefined }); Object.defineProperty(navigator, 'plugins', { get: () => [1, 2, 3, 4, 5] }); Object.defineProperty(navigator, 'languages', { get: () => ['en-US', 'en'] }); @@ -307,13 +892,11 @@ class NetworkFetchMonitor: NSObject, ObservableObject { try { const fullUrl = new URL(url, window.location.href).href; - console.log('FETCH INTERCEPTED:', fullUrl); window.webkit.messageHandlers.networkLogger.postMessage({ type: 'fetch', url: fullUrl }); } catch(e) { - console.log('FETCH INTERCEPTED (fallback):', url); window.webkit.messageHandlers.networkLogger.postMessage({ type: 'fetch', url: url.toString() @@ -332,7 +915,6 @@ class NetworkFetchMonitor: NSObject, ObservableObject { this._url = url; } - console.log('XHR OPEN:', this._url); window.webkit.messageHandlers.networkLogger.postMessage({ type: 'xhr-open', url: this._url @@ -344,7 +926,6 @@ class NetworkFetchMonitor: NSObject, ObservableObject { this.onreadystatechange = function() { if (this.readyState === 4) { if (this.responseURL) { - console.log('XHR RESPONSE URL:', this.responseURL); window.webkit.messageHandlers.networkLogger.postMessage({ type: 'xhr-response', url: this.responseURL @@ -358,7 +939,6 @@ class NetworkFetchMonitor: NSObject, ObservableObject { const matches = responseText.match(urlRegex); if (matches) { matches.forEach(function(match) { - console.log('URL IN RESPONSE:', match); window.webkit.messageHandlers.networkLogger.postMessage({ type: 'response-content', url: match @@ -367,7 +947,6 @@ class NetworkFetchMonitor: NSObject, ObservableObject { } } } catch(e) { - console.log('Response text check failed:', e); } } @@ -381,7 +960,6 @@ class NetworkFetchMonitor: NSObject, ObservableObject { XMLHttpRequest.prototype.send = function() { if (this._url) { - console.log('XHR SEND:', this._url); window.webkit.messageHandlers.networkLogger.postMessage({ type: 'xhr-send', url: this._url @@ -392,7 +970,6 @@ class NetworkFetchMonitor: NSObject, ObservableObject { const originalWebSocket = window.WebSocket; window.WebSocket = function(url, protocols) { - console.log('WEBSOCKET:', url); window.webkit.messageHandlers.networkLogger.postMessage({ type: 'websocket', url: url @@ -410,7 +987,6 @@ class NetworkFetchMonitor: NSObject, ObservableObject { Object.defineProperty(obj.prototype, prop, { set: function(value) { if (typeof value === 'string' && (value.includes('http') || value.includes('.m3u8') || value.includes('.ts'))) { - console.log('URL PROPERTY SET:', prop, value); window.webkit.messageHandlers.networkLogger.postMessage({ type: 'property-set', url: value @@ -434,26 +1010,19 @@ class NetworkFetchMonitor: NSObject, ObservableObject { let jwHookAttempts = 0; const aggressiveJWHook = function() { jwHookAttempts++; - console.log('JWPlayer hook attempt:', jwHookAttempts); if (window.jwplayer) { - console.log('JWPlayer detected!'); - const originalJWPlayer = window.jwplayer; window.jwplayer = function(id) { - console.log('JWPlayer called with ID:', id); const player = originalJWPlayer.apply(this, arguments); if (player && player.setup) { const originalSetup = player.setup; player.setup = function(config) { - console.log('JWPlayer setup config:', config); - const extractUrls = function(obj, path = '') { if (!obj) return; if (typeof obj === 'string' && (obj.includes('http') || obj.includes('.m3u8') || obj.includes('.ts'))) { - console.log('JWPlayer URL found at', path + ':', obj); window.webkit.messageHandlers.networkLogger.postMessage({ type: 'jwplayer-config', url: obj @@ -508,7 +1077,6 @@ class NetworkFetchMonitor: NSObject, ObservableObject { const element = document.querySelector(selector); if (element && element.offsetParent !== null) { results.waitResults[selector] = true; - console.log('Element found and visible:', selector); } }); @@ -527,7 +1095,6 @@ class NetworkFetchMonitor: NSObject, ObservableObject { try { element.click(); clicked = true; - console.log('Successfully clicked:', selector); } catch(e1) { try { const event = new MouseEvent('click', { @@ -537,9 +1104,7 @@ class NetworkFetchMonitor: NSObject, ObservableObject { }); element.dispatchEvent(event); clicked = true; - console.log('Successfully dispatched click:', selector); } catch(e2) { - console.log('Failed to click element:', selector, e2); } } } @@ -551,7 +1116,6 @@ class NetworkFetchMonitor: NSObject, ObservableObject { elementsFound: elements.length }); } catch(e) { - console.log('Error clicking selector:', selector, e); results.clickResults.push({ selector: selector, success: false, @@ -576,13 +1140,10 @@ class NetworkFetchMonitor: NSObject, ObservableObject { }; const nuclearScan = function() { - console.log('Nuclear scan initiated'); - Object.keys(window).forEach(function(key) { try { const value = window[key]; if (typeof value === 'string' && (value.includes('.m3u8') || value.includes('.ts') || (value.includes('http') && value.includes('.')))) { - console.log('Global URL found:', key, value); window.webkit.messageHandlers.networkLogger.postMessage({ type: 'global-variable', url: value @@ -598,7 +1159,6 @@ class NetworkFetchMonitor: NSObject, ObservableObject { const matches = script.textContent.match(urlRegex); if (matches) { matches.forEach(function(match) { - console.log('URL in script:', match); window.webkit.messageHandlers.networkLogger.postMessage({ type: 'script-content', url: match @@ -613,7 +1173,28 @@ class NetworkFetchMonitor: NSObject, ObservableObject { setTimeout(nuclearScan, 1500); setTimeout(nuclearScan, 3000); - console.log('Advanced interceptor setup complete'); + window.captureCookies = function() { + const cookies = {}; + document.cookie.split(';').forEach(function(cookie) { + const parts = cookie.trim().split('='); + if (parts.length === 2) { + cookies[parts[0]] = decodeURIComponent(parts[1]); + } + }); + + if (Object.keys(cookies).length > 0) { + window.webkit.messageHandlers.networkLogger.postMessage({ + type: 'cookies', + cookies: cookies + }); + } + + return cookies; + }; + + setTimeout(window.captureCookies, 1000); + setTimeout(window.captureCookies, 3000); + setTimeout(window.captureCookies, 5000); })(); """ @@ -627,6 +1208,23 @@ class NetworkFetchMonitor: NSObject, ObservableObject { webView?.customUserAgent = URLSession.randomUserAgent } + private func loadHTMLContent(_ htmlContent: String) { + guard let webView = webView else { return } + + addRequest("data:text/html;charset=utf-8,") + + webView.loadHTMLString(htmlContent, baseURL: nil) + + DispatchQueue.main.asyncAfter(deadline: .now() + 2.0) { + self.performCustomInteractions() + + if self.options?.returnCookies == true { + self.captureCookies { + } + } + } + } + private func loadURL(url: URL, headers: [String: String]) { guard let webView = webView else { return } @@ -645,7 +1243,6 @@ class NetworkFetchMonitor: NSObject, ObservableObject { for (key, value) in headers { request.setValue(value, forHTTPHeaderField: key) - Logger.shared.log("Custom header set: \(key): \(value)", type: "Debug") } if request.value(forHTTPHeaderField: "Referer") == nil { @@ -664,9 +1261,12 @@ class NetworkFetchMonitor: NSObject, ObservableObject { DispatchQueue.main.asyncAfter(deadline: .now() + 2.0) { self.performCustomInteractions() + + if self.options?.returnCookies == true { + self.captureCookies { + } + } } - - Logger.shared.log("Started loading: \(url.absoluteString)", type: "Debug") } private func performCustomInteractions() { @@ -681,21 +1281,9 @@ class NetworkFetchMonitor: NSObject, ObservableObject { [\(waitSelectorsJS)], [\(clickSelectorsJS)], \(options.maxWaitTime) - ).then(function(results) { - console.log('Custom interaction completed:', results); - }); + ); """ - - statusMessage = "Performing custom interactions..." - Logger.shared.log("NetworkFetch: Starting custom interactions - wait for: \(options.waitForSelectors), click: \(options.clickSelectors)", type: "Debug") - - webView.evaluateJavaScript(customInteractionJS) { result, error in - if let error = error { - Logger.shared.log("NetworkFetch: Custom interaction error: \(error)", type: "Error") - } else { - Logger.shared.log("NetworkFetch: Custom interaction JavaScript executed successfully", type: "Debug") - } - } + webView.evaluateJavaScript(customInteractionJS, completionHandler: nil) } else { simulateUserInteraction() } @@ -716,31 +1304,23 @@ class NetworkFetchMonitor: NSObject, ObservableObject { id.toLowerCase().includes('play') || el.getAttribute('aria-label')?.toLowerCase().includes('play'); }); - filteredButtons.forEach(function(btn, index) { setTimeout(function() { try { btn.click(); - console.log('Clicked play button:', btn); - } catch(e) { - console.log('Failed to click button:', e); - } + } catch(e) {} }, index * 200); }); - window.scrollTo(0, document.body.scrollHeight / 2); setTimeout(function() { window.scrollTo(0, 0); }, 500); - document.querySelectorAll('video').forEach(function(video) { if (video.play && typeof video.play === 'function') { video.play().catch(function(e) { - console.log('Could not autoplay video:', e); }); } }); - if (window.jwplayer) { try { const players = window.jwplayer().getInstances?.() || []; @@ -749,11 +1329,8 @@ class NetworkFetchMonitor: NSObject, ObservableObject { player.play(); } }); - } catch(e) { - console.log('JW Player interaction failed:', e); - } + } catch(e) {} } - if (window.videojs) { try { window.videojs.getAllPlayers?.().forEach(function(player) { @@ -761,18 +1338,11 @@ class NetworkFetchMonitor: NSObject, ObservableObject { player.play(); } }); - } catch(e) { - console.log('Video.js interaction failed:', e); - } + } catch(e) {} } }, 1000); """ - - webView.evaluateJavaScript(jsInteraction) { result, error in - if let error = error { - Logger.shared.log("JavaScript interaction error: \(error)", type: "Error") - } - } + webView.evaluateJavaScript(jsInteraction, completionHandler: nil) } private func stopMonitoring(reason: String = "completed") { @@ -782,44 +1352,36 @@ class NetworkFetchMonitor: NSObject, ObservableObject { webView?.stopLoading() webView?.configuration.userContentController.removeScriptMessageHandler(forName: "networkLogger") + let originalUrl = options?.htmlContent != nil ? "data:text/html;charset=utf-8," : (webView?.url?.absoluteString ?? "") + let result: [String: Any] = [ - "originalUrl": webView?.url?.absoluteString ?? "", + "originalUrl": originalUrl, "requests": networkRequests, "html": htmlContent as Any, + "cookies": cookies.isEmpty ? NSNull() : cookies, "success": true, "cutoffTriggered": cutoffTriggered, "cutoffUrl": cutoffUrl as Any, "htmlCaptured": htmlCaptured, + "cookiesCaptured": cookiesCaptured, "elementsClicked": elementsClicked, "waitResults": waitResults ] webView = nil - if cutoffTriggered { - statusMessage = "Cutoff triggered! Found \(networkRequests.count) requests" - Logger.shared.log("NetworkFetch stopped early due to cutoff: \(cutoffUrl ?? "unknown")", type: "Debug") - } else if htmlCaptured { - statusMessage = "HTML captured! Found \(networkRequests.count) requests, clicked \(elementsClicked.count) elements" - } else { - statusMessage = "Completed! Found \(networkRequests.count) requests, clicked \(elementsClicked.count) elements" - } completionHandler?(result) completionHandler = nil - - Logger.shared.log("Monitoring stopped (\(reason)). Total requests: \(networkRequests.count), HTML captured: \(htmlCaptured), Elements clicked: \(elementsClicked.count)", type: "Debug") } private func addRequest(_ urlString: String) { DispatchQueue.main.async { if !self.networkRequests.contains(urlString) { self.networkRequests.append(urlString) - Logger.shared.log("Captured: \(urlString)", type: "Debug") if let cutoff = self.options?.cutoff, !cutoff.isEmpty { if urlString.lowercased().contains(cutoff.lowercased()) { - Logger.shared.log("Cutoff triggered by: \(urlString)", type: "Debug") self.cutoffTriggered = true self.cutoffUrl = urlString self.stopMonitoring(reason: "cutoff") @@ -850,24 +1412,33 @@ extension NetworkFetchMonitor: WKScriptMessageHandler { if let messageBody = message.body as? [String: Any] { if let url = messageBody["url"] as? String { addRequest(url) - } else if let type = messageBody["type"] as? String, type == "click-results" { - if let results = messageBody["results"] as? [String: Any] { - if let clickResults = results["clickResults"] as? [[String: Any]] { - DispatchQueue.main.async { - for clickResult in clickResults { - if let selector = clickResult["selector"] as? String, - let success = clickResult["success"] as? Bool, success { - self.elementsClicked.append(selector) - Logger.shared.log("NetworkFetch: Successfully clicked element: \(selector)", type: "Debug") + } else if let type = messageBody["type"] as? String { + if type == "click-results" { + if let results = messageBody["results"] as? [String: Any] { + if let clickResults = results["clickResults"] as? [[String: Any]] { + DispatchQueue.main.async { + for clickResult in clickResults { + if let selector = clickResult["selector"] as? String, + let success = clickResult["success"] as? Bool, success { + self.elementsClicked.append(selector) + } } } } + + if let waitResults = results["waitResults"] as? [String: Bool] { + DispatchQueue.main.async { + self.waitResults = waitResults + } + } } - - if let waitResults = results["waitResults"] as? [String: Bool] { + } else if type == "cookies" { + if let cookiesData = messageBody["cookies"] as? [String: String] { DispatchQueue.main.async { - self.waitResults = waitResults - Logger.shared.log("NetworkFetch: Wait results: \(waitResults)", type: "Debug") + for (key, value) in cookiesData { + self.cookies[key] = value + } + self.cookiesCaptured = !self.cookies.isEmpty } } }