Creepso/mangayomi
1
0
Fork 0
Code Issues Pull requests Projects Releases 1 Packages Wiki Activity Actions

fix madara chapter list parse crash on windows

Browse source
This commit is contained in:
kodjomoustapha 2023-06-20 16:04:16 +01:00
parent 616fb0699d
commit eb6fa6d26f
11 changed files with 124 additions and 165 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
lib/modules/webview/webview.dart
View file

@ -9,6 +9,7 @@ import 'package:flutter_inappwebview/flutter_inappwebview.dart';
import 'package:flutter_riverpod/flutter_riverpod.dart';
import 'package:mangayomi/main.dart';
import 'package:mangayomi/models/settings.dart';
import 'package:mangayomi/services/http_service/cloudflare/cloudflare_bypass.dart';
import 'package:mangayomi/services/http_service/cloudflare/cookie.dart';
import 'package:mangayomi/utils/constant.dart';
import 'package:url_launcher/url_launcher.dart';
@ -238,6 +239,7 @@ class _MangaWebViewState extends ConsumerState<MangaWebView> {
});
},
initialSettings: InAppWebViewSettings(
contentBlockers: adsContentBlockers(),
userAgent: isar.settings.getSync(227)!.userAgent!),
initialUrlRequest:
URLRequest(url: WebUri.uri(Uri.parse(widget.url))),

77
lib/services/http_service/cloudflare/cloudflare_bypass.dart
View file

@ -45,18 +45,10 @@ Future<dom.Document?> cloudflareBypassDom(CloudflareBypassDomRef ref,
});
html = await decodeHtml(webview);
htmll = dom.Document.html(html!);
isOk = true;
webview.close();
await Future.doWhile(() async {
await Future.delayed(const Duration(seconds: 1));
if (isOk == true) {
return false;
}
return true;
});
} else {
HeadlessInAppWebView? headlessWebViewJapScan;
headlessWebViewJapScan = HeadlessInAppWebView(
HeadlessInAppWebView? headlessWebView;
headlessWebView = HeadlessInAppWebView(
onLoadStop: (controller, u) async {
String? html;
html = await controller.evaluateJavascript(
@ -65,6 +57,7 @@ Future<dom.Document?> cloudflareBypassDom(CloudflareBypassDomRef ref,
await Future.doWhile(() async {
if (html == null ||
html!.contains("Just a moment") ||
html!.contains("Un instant…") ||
html!.contains("https://challenges.cloudflare.com")) {
html = await controller.evaluateJavascript(
source:
@ -78,16 +71,17 @@ Future<dom.Document?> cloudflareBypassDom(CloudflareBypassDomRef ref,
"window.document.getElementsByTagName('html')[0].outerHTML;");
htmll = dom.Document.html(html!);
isOk = true;
headlessWebViewJapScan!.dispose();
headlessWebView!.dispose();
},
initialSettings:
useUserAgent ? InAppWebViewSettings(userAgent: ua) : null,
initialSettings: InAppWebViewSettings(
userAgent: useUserAgent ? ua : "",
),
initialUrlRequest: URLRequest(
url: WebUri.uri(Uri.parse(url)),
),
);
headlessWebViewJapScan.run();
headlessWebView.run();
await Future.doWhile(() async {
await Future.delayed(const Duration(seconds: 1));
@ -134,8 +128,8 @@ Future<String> cloudflareBypassHtml(CloudflareBypassHtmlRef ref,
isOk = true;
webview.close();
} else {
HeadlessInAppWebView? headlessWebViewJapScan;
headlessWebViewJapScan = HeadlessInAppWebView(
HeadlessInAppWebView? headlessWebView;
headlessWebView = HeadlessInAppWebView(
onLoadStop: (controller, u) async {
html = await controller.evaluateJavascript(
source:
@ -143,7 +137,8 @@ Future<String> cloudflareBypassHtml(CloudflareBypassHtmlRef ref,
await Future.doWhile(() async {
if (html == null ||
html!.contains("Just a moment") ||
html!.contains("Un instant…")) {
html!.contains("Un instant…") ||
html!.contains("https://challenges.cloudflare.com")) {
html = await controller.evaluateJavascript(
source:
"window.document.getElementsByTagName('html')[0].outerHTML;");
@ -156,16 +151,17 @@ Future<String> cloudflareBypassHtml(CloudflareBypassHtmlRef ref,
source:
"window.document.getElementsByTagName('html')[0].outerHTML;");
isOk = true;
headlessWebViewJapScan!.dispose();
headlessWebView!.dispose();
},
initialSettings:
useUserAgent ? InAppWebViewSettings(userAgent: ua) : null,
initialSettings: InAppWebViewSettings(
userAgent: useUserAgent ? ua : "",
),
initialUrlRequest: URLRequest(
url: WebUri.uri(Uri.parse(url)),
),
);
headlessWebViewJapScan.run();
headlessWebView.run();
await Future.doWhile(() async {
await Future.delayed(const Duration(seconds: 1));
if (isOk == true) {
@ -177,3 +173,42 @@ Future<String> cloudflareBypassHtml(CloudflareBypassHtmlRef ref,
}
return html!;
}
List<ContentBlocker> adsContentBlockers() {
final List<ContentBlocker> contentBlockers = [];
// list of Ad URL filters to be used to block ads loading.
final adUrlFilters = [
".*.doubleclick.net/.*",
".*.ads.pubmatic.com/.*",
".*.googlesyndication.com/.*",
".*.google-analytics.com/.*",
".*.adservice.google.*/.*",
".*.adbrite.com/.*",
".*.exponential.com/.*",
".*.quantserve.com/.*",
".*.scorecardresearch.com/.*",
".*.zedo.com/.*",
".*.adsafeprotected.com/.*",
".*.teads.tv/.*",
".*.outbrain.com/.*"
];
for (final adUrlFilter in adUrlFilters) {
contentBlockers.add(ContentBlocker(
trigger: ContentBlockerTrigger(
urlFilter: adUrlFilter,
),
action: ContentBlockerAction(
type: ContentBlockerActionType.BLOCK,
)));
}
// apply the "display: none" style to some HTML elements
contentBlockers.add(ContentBlocker(
trigger: ContentBlockerTrigger(
urlFilter: ".*",
),
action: ContentBlockerAction(
type: ContentBlockerActionType.CSS_DISPLAY_NONE,
selector: ".banner, .banners, .ads, .ad, .advert")));
return contentBlockers;
}

8
lib/services/http_service/http_res_to_dom_html.dart
View file

@ -1,8 +0,0 @@
import 'package:html/dom.dart';
import 'package:http/http.dart' as http;
Future<Document> httpResToDom(
{required String url, required Map<String, String>? headers}) async {
final response = await http.get(Uri.parse(url), headers: headers);
return Document.html(response.body);
}

7
lib/services/http_service/http_service.dart
View file

@ -1,6 +1,5 @@
import 'package:html/dom.dart';
import 'package:mangayomi/services/http_service/cloudflare/cloudflare_bypass.dart';
import 'package:mangayomi/services/http_service/http_res_to_dom_html.dart';
import 'package:http/http.dart' as http;
import 'package:mangayomi/sources/utils/utils.dart';
import 'package:mangayomi/utils/headers.dart';
@ -45,3 +44,9 @@ Future<dynamic> httpGet(HttpGetRef ref,
return resHtml;
}
}
Future<Document> httpResToDom(
{required String url, required Map<String, String>? headers}) async {
final response = await http.get(Uri.parse(url), headers: headers);
return Document.html(response.body);
}

2
lib/sources/multisrc/heancms/heancms.dart
View file

@ -1,7 +1,7 @@
import 'dart:convert';
import 'package:flutter_riverpod/flutter_riverpod.dart';
import 'package:mangayomi/models/chapter.dart';
import 'package:mangayomi/services/http_service/http_res_to_dom_html.dart';
import 'package:mangayomi/services/http_service/http_service.dart';
import 'package:mangayomi/sources/multisrc/heancms/model/search.dart';
import 'package:mangayomi/sources/multisrc/heancms/utils/utils.dart';
import 'package:mangayomi/sources/service.dart';

50
lib/sources/multisrc/madara/madara_source_list.dart
View file

@ -77,15 +77,15 @@ List<Source> _madaraSourcesList = [
dateFormat: "MMM dd, yyyy",
dateFormatLocale: defaultDateFormatLocale,
),
Source(
sourceName: "Clover Manga",
baseUrl: "https://clover-manga.com",
lang: "tr",
typeSource: TypeSource.madara,
logoUrl: '',
dateFormat: defaultDateFormat,
dateFormatLocale: "tr",
),
// Source(
// sourceName: "Clover Manga",
// baseUrl: "https://clover-manga.com",
// lang: "tr",
// typeSource: TypeSource.madara,
// logoUrl: '',
// dateFormat: defaultDateFormat,
// dateFormatLocale: "tr",
// ),
Source(
sourceName: "CookieToon",
baseUrl: "https://cookietoon.online",
@ -374,7 +374,7 @@ List<Source> _madaraSourcesList = [
dateFormat: "MMMM d, yyyy",
dateFormatLocale: defaultDateFormatLocale,
),
Source(
sourceName: "Manhwua.fans",
baseUrl: "https://manhwua.fans",
@ -525,15 +525,15 @@ List<Source> _madaraSourcesList = [
dateFormat: "dd/MM/yyyy",
dateFormatLocale: "pt-BR",
),
Source(
sourceName: "365Manga",
baseUrl: "https://365manga.com",
lang: "en",
typeSource: TypeSource.madara,
logoUrl: '',
dateFormat: defaultDateFormat,
dateFormatLocale: defaultDateFormatLocale,
),
// Source(
// sourceName: "365Manga",
// baseUrl: "https://365manga.com",
// lang: "en",
// typeSource: TypeSource.madara,
// logoUrl: '',
// dateFormat: defaultDateFormat,
// dateFormatLocale: defaultDateFormatLocale,
// ),
Source(
sourceName: "Tortuga Ceviri",
baseUrl: "https://tortuga-ceviri.com",
@ -581,4 +581,14 @@ List<Source> _madaraSourcesList = [
dateFormat: "dd/MM/yyyy",
dateFormatLocale: "pt-BR",
),
];
// Source(
// sourceName: "مانجا ليك",
// baseUrl: "https://mangalek.com",
// lang: "ar",
// typeSource: TypeSource.madara,
// logoUrl: '',
// dateFormat: "MMMM dd, yyyy",
// dateFormatLocale: "ar",
// ),
];

120
lib/sources/multisrc/madara/src/madara.dart
View file

@ -1,18 +1,14 @@
import 'dart:io';
import 'package:desktop_webview_window/desktop_webview_window.dart';
import 'package:flutter/material.dart';
import 'package:flutter_inappwebview/flutter_inappwebview.dart';
import 'package:flutter_riverpod/flutter_riverpod.dart';
import 'package:html/dom.dart';
import 'package:http/http.dart';
import 'package:mangayomi/models/chapter.dart';
import 'package:mangayomi/modules/webview/webview.dart';
import 'package:mangayomi/services/http_service/http_service.dart';
import 'package:mangayomi/sources/multisrc/madara/src/utils.dart';
import 'package:mangayomi/sources/service.dart';
import 'package:mangayomi/sources/utils/utils.dart';
import 'package:mangayomi/utils/constant.dart';
import 'package:mangayomi/utils/reg_exp_matcher.dart';
import 'package:mangayomi/utils/xpath_selector.dart';
import 'package:http/http.dart' as http;
class Madara extends MangaYomiServices {
@override
@ -91,101 +87,34 @@ class Madara extends MangaYomiServices {
.querySelectorAll("div.genres-content a")
.map((e) => e.text)
.toList();
bool isOk = false;
String? html;
if (Platform.isWindows || Platform.isLinux) {
final webview = await WebviewWindow.create(
configuration: CreateConfiguration(
windowHeight: 500,
windowWidth: 500,
userDataFolderWindows: await getWebViewPath(),
),
);
webview
..setBrightness(Brightness.dark)
..setApplicationNameForUserAgent(defaultUserAgent)
..launch(manga.url!);
final mangaId = dom
.querySelectorAll("div[id^=manga-chapters-holder]")
.map((e) => e.attributes['data-id'])
.first;
Response? mangaResponse;
final headers = {
"Referer": "${getMangaBaseUrl(source)}/",
"Content-Type": "application/x-www-form-urlencoded",
"X-Requested-With": "XMLHttpRequest"
};
mangaResponse = await http.post(
Uri.parse(
"${getMangaBaseUrl(source)}/wp-admin/admin-ajax.php?action=manga_get_chapters&manga=$mangaId"),
headers: headers);
await Future.doWhile(() async {
await Future.delayed(const Duration(seconds: 10));
html = await decodeHtml(webview);
if (xpathSelector(html!)
.query("//*[@id='manga-chapters-holder']/div[2]/div/ul/li/a/@href")
.attrs
.isEmpty) {
html = await decodeHtml(webview);
return true;
}
return false;
});
html = await decodeHtml(webview);
isOk = true;
await Future.doWhile(() async {
await Future.delayed(const Duration(seconds: 1));
if (isOk == true) {
return false;
}
return true;
});
html = await decodeHtml(webview);
webview.close();
} else {
HeadlessInAppWebView? headlessWebViewJapScan;
headlessWebViewJapScan = HeadlessInAppWebView(
onLoadStop: (controller, u) async {
html = await controller.evaluateJavascript(
source:
"window.document.getElementsByTagName('html')[0].outerHTML;");
await Future.doWhile(() async {
html = await controller.evaluateJavascript(
source:
"window.document.getElementsByTagName('html')[0].outerHTML;");
if (xpathSelector(html!)
.query(
"//*[@id='manga-chapters-holder']/div[2]/div/ul/li/a/@href")
.attrs
.isEmpty) {
html = await controller.evaluateJavascript(
source:
"window.document.getElementsByTagName('html')[0].outerHTML;");
return true;
}
return false;
});
html = await controller.evaluateJavascript(
source:
"window.document.getElementsByTagName('html')[0].outerHTML;");
isOk = true;
headlessWebViewJapScan!.dispose();
},
initialUrlRequest: URLRequest(
url: WebUri.uri(Uri.parse(manga.url!)),
),
);
headlessWebViewJapScan.run();
await Future.doWhile(() async {
await Future.delayed(const Duration(seconds: 1));
if (isOk == true) {
return false;
}
return true;
});
if (mangaResponse.statusCode == 400) {
mangaResponse = await http.post(Uri.parse("${manga.url}ajax/chapters"),
headers: headers);
}
final xpath = xpathSelector(html!);
for (var url in xpath
.query("//*[@id='manga-chapters-holder']/div[2]/div/ul/li/a/@href")
.attrs) {
final xpath = xpathSelector(mangaResponse.body);
for (var url in xpath.query("//li/a/@href").attrs) {
chapterUrl.add(url!);
}
for (var title in xpath
.query("//*[@id='manga-chapters-holder']/div[2]/div/ul/li/a/text()")
.attrs) {
for (var title in xpath.query("//li/a/text()").attrs) {
chapterTitle.add(title!.trim().trimLeft().trimRight());
}
final dateF = xpath
.query(
"//*[@id='manga-chapters-holder']/div[2]/div/ul/li/span/i/text()")
.attrs;
final dateF = xpath.query("//li/span/i/text()").attrs;
if (dateF.length == chapterUrl.length) {
for (var date in dateF) {
@ -200,7 +129,6 @@ class Madara extends MangaYomiServices {
chapterDate.add(parseChapterDate(date!, source).toString());
}
}
return mangadetailRes(manga: manga, source: source);
}

7
lib/sources/src/fr/japscan/src/japscan.dart
View file

@ -118,10 +118,7 @@ class Japscan extends MangaYomiServices {
required String lang,
required AutoDisposeFutureProviderRef ref}) async {
final dom = await ref.watch(httpGetProvider(
useUserAgent: true,
url: "${getMangaBaseUrl(source)}/",
source: source,
resDom: true)
url: "${getMangaBaseUrl(source)}/", source: source, resDom: true)
.future) as Document?;
if (dom!.querySelectorAll('#top_mangas_week > ul > li ').isNotEmpty) {
final urls = dom
@ -129,6 +126,7 @@ class Japscan extends MangaYomiServices {
.where((e) => e.attributes['href'].toString().contains('manga'))
.map((e) => e.attributes['href'])
.toList();
print(urls);
for (var ok in urls) {
url.add("${getMangaBaseUrl(source)}$ok");
}
@ -151,7 +149,6 @@ class Japscan extends MangaYomiServices {
required String lang,
required AutoDisposeFutureProviderRef ref}) async {
final dom = await ref.watch(httpGetProvider(
useUserAgent: true,
url:
"https://www.google.com/search?q=${query.toLowerCase()}+japscan",
source: source,

1
lib/sources/src/fr/mangakawaii/src/mangakawaii.dart
View file

@ -2,7 +2,6 @@ import 'package:flutter_riverpod/flutter_riverpod.dart';
import 'package:html/dom.dart';
import 'package:mangayomi/models/chapter.dart';
import 'package:mangayomi/models/manga.dart';
import 'package:mangayomi/services/http_service/http_res_to_dom_html.dart';
import 'package:mangayomi/services/http_service/http_service.dart';
import 'package:mangayomi/sources/service.dart';
import 'package:mangayomi/sources/utils/utils.dart';

12
pubspec.lock
View file

@ -418,10 +418,10 @@ packages:
dependency: "direct main"
description:
name: flutter_inappwebview
sha256: "6d6c741ddba1dba5229d63ba75767064791a7ce845196b45e31105e93d67c949"
sha256: fad1f2740ff4b5b7da378a639f54beeb9d787b6339c89a9de00494d92372c0bb
url: "https://pub.dev"
source: hosted
version: "6.0.0-beta.22"
version: "6.0.0-beta.24+1"
flutter_inappwebview_internal_annotations:
dependency: transitive
description:
@ -949,14 +949,6 @@ packages:
url: "https://pub.dev"
source: hosted
version: "0.27.7"
scrollable_positioned_list:
dependency: "direct main"
description:
name: scrollable_positioned_list
sha256: ca7fcaa743db712d4f7b1580526f494d0093c77a721a65705ee51fbeac7a2bd3
url: "https://pub.dev"
source: hosted
version: "0.3.5"
scrollview_observer:
dependency: "direct main"
description:

3
pubspec.yaml
View file

@ -42,7 +42,6 @@ dependencies:
font_awesome_flutter: ^10.1.0
expandable_text: ^2.3.0
flex_color_scheme: ^7.1.2
scrollable_positioned_list: ^0.3.5
extended_image: ^ 8.0.2
photo_view: ^0.14.0
draggable_scrollbar: ^0.1.0
@ -53,7 +52,7 @@ dependencies:
package_info_plus: ^4.0.2
background_downloader: ^7.4.0
permission_handler: ^10.2.0
flutter_inappwebview: ^6.0.0-beta.22
flutter_inappwebview: ^6.0.0-beta.24+1
draggable_menu: ^2.0.2
isar: 3.1.0+1
isar_flutter_libs: 3.1.0+1