From 33eb2458a4dec18369d5515ab8ac45dc9eb97fc9 Mon Sep 17 00:00:00 2001 From: kodjomoustapha <107993382+kodjodevf@users.noreply.github.com> Date: Thu, 25 May 2023 20:59:02 +0100 Subject: [PATCH] initial adding madara multi source --- lib/models/source.dart | 4 +- lib/services/get_chapter_url.dart | 8 + lib/services/get_manga_detail.dart | 10 + lib/services/get_popular_manga.dart | 10 +- .../cloudflare/cloudflare_bypass.dart | 1 + .../multisrc/madara/madara_source_list.dart | 29 +++ lib/sources/multisrc/madara/src/madara.dart | 216 +++++++++++++++--- .../mangathemesia/src/mangathemesia.dart | 2 +- lib/sources/source_list.dart | 5 +- lib/utils/reg_exp_matcher.dart | 7 + lib/utils/xpath_selector.dart | 7 + pubspec.lock | 32 +++ pubspec.yaml | 1 + 13 files changed, 300 insertions(+), 32 deletions(-) create mode 100644 lib/utils/xpath_selector.dart diff --git a/lib/models/source.dart b/lib/models/source.dart index 4868c0e..d98a82c 100644 --- a/lib/models/source.dart +++ b/lib/models/source.dart @@ -60,5 +60,7 @@ enum TypeSource { mmrcms, - heancms + heancms, + + madara } diff --git a/lib/services/get_chapter_url.dart b/lib/services/get_chapter_url.dart index 9298974..f08469d 100644 --- a/lib/services/get_chapter_url.dart +++ b/lib/services/get_chapter_url.dart @@ -7,6 +7,7 @@ import 'package:mangayomi/models/settings.dart'; import 'package:mangayomi/models/source.dart'; import 'package:mangayomi/providers/storage_provider.dart'; import 'package:mangayomi/sources/multisrc/heancms/heancms.dart'; +import 'package:mangayomi/sources/multisrc/madara/src/madara.dart'; import 'package:mangayomi/sources/src/all/comick/src/comick.dart'; import 'package:mangayomi/sources/src/en/mangahere/src/mangahere.dart'; import 'package:mangayomi/sources/src/fr/japscan/src/japscan.dart'; @@ -104,6 +105,13 @@ Future getChapterUrl( else if (getMangaTypeSource(source) == TypeSource.heancms) { pageUrls = await HeanCms().getChapterUrl(chapter: chapter, ref: ref); } + /***********/ + /*madara*/ + /***********/ + + else if (getMangaTypeSource(source) == TypeSource.madara) { + pageUrls = await Madara().getChapterUrl(chapter: chapter, ref: ref); + } if (pageUrls.isNotEmpty) { if (!incognitoMode) { diff --git a/lib/services/get_manga_detail.dart b/lib/services/get_manga_detail.dart index b26b1d5..af665a5 100644 --- a/lib/services/get_manga_detail.dart +++ b/lib/services/get_manga_detail.dart @@ -1,6 +1,7 @@ import 'dart:async'; import 'package:mangayomi/models/source.dart'; import 'package:mangayomi/sources/multisrc/heancms/heancms.dart'; +import 'package:mangayomi/sources/multisrc/madara/src/madara.dart'; import 'package:mangayomi/sources/service.dart'; import 'package:mangayomi/sources/src/all/comick/src/comick.dart'; import 'package:mangayomi/sources/src/en/mangahere/src/mangahere.dart'; @@ -77,5 +78,14 @@ Future getMangaDetail(GetMangaDetailRef ref, mangadetail = await HeanCms() .getMangaDetail(manga: manga, lang: lang, source: source, ref: ref); } + + /***********/ + /*madara*/ + /***********/ + + else if (getMangaTypeSource(source.toLowerCase()) == TypeSource.madara) { + mangadetail = await Madara() + .getMangaDetail(manga: manga, lang: lang, source: source, ref: ref); + } return mangadetail!; } diff --git a/lib/services/get_popular_manga.dart b/lib/services/get_popular_manga.dart index a2e0c27..ae1fd8e 100644 --- a/lib/services/get_popular_manga.dart +++ b/lib/services/get_popular_manga.dart @@ -1,6 +1,7 @@ import 'dart:async'; import 'package:mangayomi/models/source.dart'; import 'package:mangayomi/sources/multisrc/heancms/heancms.dart'; +import 'package:mangayomi/sources/multisrc/madara/src/madara.dart'; import 'package:mangayomi/sources/service.dart'; import 'package:mangayomi/sources/src/all/comick/src/comick.dart'; import 'package:mangayomi/sources/src/en/mangahere/src/mangahere.dart'; @@ -67,11 +68,18 @@ Future> getPopularManga(GetPopularMangaRef ref, } /***********/ - /*japscan*/ + /*heancms*/ /***********/ else if (getMangaTypeSource(source) == TypeSource.heancms) { popularManga = await HeanCms().getPopularManga(source: source, page: page, ref: ref); } + /***********/ + /*madara*/ + /***********/ + else if (getMangaTypeSource(source) == TypeSource.madara) { + popularManga = + await Madara().getPopularManga(source: source, page: page, ref: ref); + } return popularManga!; } diff --git a/lib/services/http_service/cloudflare/cloudflare_bypass.dart b/lib/services/http_service/cloudflare/cloudflare_bypass.dart index 30a6f51..1fdf146 100644 --- a/lib/services/http_service/cloudflare/cloudflare_bypass.dart +++ b/lib/services/http_service/cloudflare/cloudflare_bypass.dart @@ -82,6 +82,7 @@ Future cloudflareBypassHtml(CloudflareBypassHtmlRef ref, } return false; }); + await Future.delayed(Duration(seconds: 10)); html = await controller.evaluateJavascript( source: "window.document.getElementsByTagName('html')[0].outerHTML;"); isOk = true; diff --git a/lib/sources/multisrc/madara/madara_source_list.dart b/lib/sources/multisrc/madara/madara_source_list.dart index e69de29..c4b6740 100644 --- a/lib/sources/multisrc/madara/madara_source_list.dart +++ b/lib/sources/multisrc/madara/madara_source_list.dart @@ -0,0 +1,29 @@ +import 'package:mangayomi/models/source.dart'; + +List get madaraSourcesList => _madaraSourcesList; +List _madaraSourcesList = [ + Source( + sourceName: "FR-Scan", + baseUrl: "https://fr-scan.com", + lang: "fr", + typeSource: TypeSource.madara, + logoUrl: '', + dateFormat: "MMMM d, yyyy", + dateFormatLocale: "fr"), + Source( + sourceName: "AstralManga", + baseUrl: "https://astral-manga.fr", + lang: "fr", + typeSource: TypeSource.madara, + logoUrl: '', + dateFormat: "dd/mm/yyyy", + dateFormatLocale: "fr"), + Source( + sourceName: "arabtoons", + baseUrl: "https://arabtoons.net", + lang: "ar", + typeSource: TypeSource.madara, + logoUrl: '', + dateFormat: "MMM d", + dateFormatLocale: "ar"), +]; diff --git a/lib/sources/multisrc/madara/src/madara.dart b/lib/sources/multisrc/madara/src/madara.dart index 1a24bb2..556a1ae 100644 --- a/lib/sources/multisrc/madara/src/madara.dart +++ b/lib/sources/multisrc/madara/src/madara.dart @@ -1,31 +1,193 @@ -// import 'package:mangayomi/models/chapter.dart'; -// import 'package:mangayomi/sources/service.dart'; +import 'dart:convert'; +import 'dart:developer'; -// class Madara extends MangaYomiServices { -// @override -// Future getChapterUrl({required Chapter chapter}) { -// // TODO: implement getChapterUrl -// throw UnimplementedError(); -// } +import 'package:flutter_inappwebview/flutter_inappwebview.dart'; +import 'package:flutter_riverpod/flutter_riverpod.dart'; +import 'package:html/dom.dart'; +import 'package:mangayomi/models/chapter.dart'; +import 'package:mangayomi/services/http_service/http_service.dart'; +import 'package:mangayomi/sources/service.dart'; +import 'package:mangayomi/sources/utils/utils.dart'; +import 'package:mangayomi/utils/reg_exp_matcher.dart'; +import 'package:mangayomi/utils/xpath_selector.dart'; -// @override -// Future getMangaDetail( -// {required GetManga manga, required String lang, required String source}) { -// // TODO: implement getMangaDetail -// throw UnimplementedError(); -// } +class Madara extends MangaYomiServices { + @override + Future> getChapterUrl( + {required Chapter chapter, + required AutoDisposeFutureProviderRef ref}) async { + final dom = await await ref.watch(httpGetProvider( + useUserAgent: true, + url: chapter.url!, + source: chapter.manga.value!.source!.toLowerCase(), + resDom: true) + .future) as Document?; + final res = dom!.querySelector( + "div.page-break, li.blocks-gallery-item, .reading-content, .text-left img"); + final imgs = res! + .querySelectorAll('img') + .map((i) => regSrcMatcher(i.outerHtml).trim().trimLeft().trimRight()) + .toList(); + if (imgs.isNotEmpty && imgs.length == 1) { + final pagesNumber = dom + .querySelector("#single-pager")! + .querySelectorAll("option") + .map((e) => e.outerHtml) + .toList(); + for (var i = 0; i < pagesNumber.length; i++) { + if (i.toString().length == 1) { + pageUrls.add( + imgs.first.replaceAll("01", '0${int.parse(i.toString()) + 1}')); + } else if (i.toString().length == 2) { + pageUrls.add( + imgs.first.replaceAll("01", '${int.parse(i.toString()) + 1}')); + } else if (i.toString().length == 3) { + pageUrls.add( + imgs.first.replaceAll("01", '${int.parse(i.toString()) + 1}')); + } + } + } else { + pageUrls = imgs; + } + log(pageUrls.toString()); -// @override -// Future> getPopularManga( -// {required String source, required int page}) { -// // TODO: implement getPopularManga -// throw UnimplementedError(); -// } + return pageUrls; + } -// @override -// Future> searchManga( -// {required String source, required String query}) { -// // TODO: implement searchManga -// throw UnimplementedError(); -// } -// } + @override + Future getMangaDetail( + {required GetManga manga, + required String lang, + required String source, + required AutoDisposeFutureProviderRef ref}) async { + final dom = await ref.watch( + httpGetProvider(url: manga.url!, source: source, resDom: true) + .future) as Document?; + author = dom! + .querySelectorAll("div.author-content > a") + .map((e) => e.text) + .toList() + .join(', '); + description = dom + .querySelectorAll( + "div.description-summary div.summary__content, div.summary_content div.post-content_item > h5 + div, div.summary_content div.manga-excerpt") + .map((e) => e.text) + .toList() + .first; + status = dom + .querySelectorAll("div.summary-content") + .map((e) => e.text.trim().trimLeft().trimRight()) + .toList() + .last; + + manga.imageUrl = dom + .querySelectorAll("div.summary_image img") + .map((e) => regSrcMatcher(e.outerHtml)) + .toList() + .first; + genre = dom + .querySelectorAll("div.genres-content a") + .map((e) => e.text) + .toList(); + bool isOk = false; + String? html; + HeadlessInAppWebView? headlessWebViewJapScan; + headlessWebViewJapScan = HeadlessInAppWebView( + onLoadStop: (controller, u) async { + html = await controller.evaluateJavascript( + source: + "window.document.getElementsByTagName('html')[0].outerHTML;"); + await Future.doWhile(() async { + html = await controller.evaluateJavascript( + source: + "window.document.getElementsByTagName('html')[0].outerHTML;"); + if (xpathSelector(html!) + .query( + "//*[@id='manga-chapters-holder']/div[2]/div/ul/li/a/@href") + .attrs + .isEmpty) { + html = await controller.evaluateJavascript( + source: + "window.document.getElementsByTagName('html')[0].outerHTML;"); + return true; + } + return false; + }); + html = await controller.evaluateJavascript( + source: + "window.document.getElementsByTagName('html')[0].outerHTML;"); + isOk = true; + headlessWebViewJapScan!.dispose(); + }, + initialUrlRequest: URLRequest( + url: WebUri.uri(Uri.parse(manga.url!)), + ), + ); + + headlessWebViewJapScan.run(); + await Future.doWhile(() async { + await Future.delayed(const Duration(seconds: 1)); + if (isOk == true) { + return false; + } + return true; + }); + final xpath = xpathSelector(html!); + for (var url in xpath + .query("//*[@id='manga-chapters-holder']/div[2]/div/ul/li/a/@href") + .attrs) { + chapterUrl.add(url!); + } + for (var title in xpath + .query("//*[@id='manga-chapters-holder']/div[2]/div/ul/li/a/text()") + .attrs) { + chapterTitle.add(title!.trim().trimLeft().trimRight()); + } + final dateF = xpath + .query( + "//*[@id='manga-chapters-holder']/div[2]/div/ul/li/span/i/text()") + .attrs; + + if (dateF.length == chapterUrl.length) { + for (var date in dateF) { + chapterDate.add(parseDate(date!, source)); + } + } else if (dateF.length < chapterUrl.length) { + final length = chapterUrl.length - dateF.length; + for (var i = 0; i < length; i++) { + chapterDate.add(DateTime.now().millisecondsSinceEpoch.toString()); + } + for (var date in dateF) { + chapterDate.add(parseDate(date!, source)); + } + } + + return mangadetailRes(manga: manga, source: source); + } + + @override + Future> getPopularManga( + {required String source, + required int page, + required AutoDisposeFutureProviderRef ref}) async { + final html = await ref.watch(httpGetProvider( + url: '${getMangaBaseUrl(source)}/manga/page/$page/?m_orderby=views', + source: source, + resDom: false) + .future) as String?; + final xpath = xpathSelector(html!); + name = xpath.query('//*[@id^="manga-item"]/a/@title').attrs; + url = xpath.query('//*[@class^="post-title"]/h3/a/@href').attrs; + image = xpath.query('//*[@id^="manga-item"]/a/img/@data-src=').attrs; + return mangaRes(); + } + + @override + Future> searchManga( + {required String source, + required String query, + required AutoDisposeFutureProviderRef ref}) { + // TODO: implement searchManga + throw UnimplementedError(); + } +} diff --git a/lib/sources/multisrc/mangathemesia/src/mangathemesia.dart b/lib/sources/multisrc/mangathemesia/src/mangathemesia.dart index ee94a7d..ae60d89 100644 --- a/lib/sources/multisrc/mangathemesia/src/mangathemesia.dart +++ b/lib/sources/multisrc/mangathemesia/src/mangathemesia.dart @@ -255,7 +255,7 @@ class MangaThemeSia extends MangaYomiServices { } final List urlss = matchess.map((m) => m.group(1)).toList(); - if (urls.length == 1 && urls.isNotEmpty) { + if (urls.isNotEmpty && urls.length == 1) { for (var i = 0; i < urlss.length; i++) { if (urlss[i]!.length == 1) { pageUrls.add( diff --git a/lib/sources/source_list.dart b/lib/sources/source_list.dart index 4fb28ce..e42a0e3 100644 --- a/lib/sources/source_list.dart +++ b/lib/sources/source_list.dart @@ -6,7 +6,7 @@ import 'package:mangayomi/sources/src/fr/japscan/japscan_source.dart'; import 'package:mangayomi/sources/src/fr/mangakawaii/mangakawaii_source.dart'; import 'package:mangayomi/sources/multisrc/mangathemesia/mangathemesia_source_list.dart'; import 'package:mangayomi/sources/multisrc/mmrcms/mmrcms_source_list.dart'; - +import 'package:mangayomi/sources/multisrc/madara/madara_source_list.dart'; List get sourcesList => _sourcesList; List _sourcesList = [ mangahereSource, @@ -15,5 +15,6 @@ List _sourcesList = [ ...comickSourcesList, ...mmrcmsSourcesList, japscanSource, - ...heanCmsSourcesList + ...heanCmsSourcesList, + ...madaraSourcesList ]; diff --git a/lib/utils/reg_exp_matcher.dart b/lib/utils/reg_exp_matcher.dart index f822373..7884021 100644 --- a/lib/utils/reg_exp_matcher.dart +++ b/lib/utils/reg_exp_matcher.dart @@ -5,6 +5,13 @@ String regHrefMatcher(String input) { return firstMatch!; } +String regDataSrcMatcher(String input) { + RegExp exp = RegExp(r'data-src="([^"]+)"'); + Iterable matches = exp.allMatches(input); + String? firstMatch = matches.first.group(1); + return firstMatch!; +} + String regSrcMatcher(String input) { RegExp exp = RegExp(r'src="([^"]+)"'); Iterable matches = exp.allMatches(input); diff --git a/lib/utils/xpath_selector.dart b/lib/utils/xpath_selector.dart new file mode 100644 index 0000000..36ed49c --- /dev/null +++ b/lib/utils/xpath_selector.dart @@ -0,0 +1,7 @@ +import 'package:html/parser.dart'; +import 'package:xpath_selector_html_parser/xpath_selector_html_parser.dart'; + +HtmlXPath xpathSelector(String html) { + final html1 = parse(html).documentElement!; + return HtmlXPath.node(html1); +} diff --git a/pubspec.lock b/pubspec.lock index 536ffaf..9f99cf7 100644 --- a/pubspec.lock +++ b/pubspec.lock @@ -306,6 +306,14 @@ packages: url: "https://pub.dev" source: hosted version: "2.3.0" + expressions: + dependency: transitive + description: + name: expressions + sha256: "75ca4343f9f8a38087bea130cf51395d737d87c6947cc19cbb8fb2732cae1a27" + url: "https://pub.dev" + source: hosted + version: "0.2.5" extended_image: dependency: "direct main" description: @@ -878,6 +886,14 @@ packages: url: "https://pub.dev" source: hosted version: "1.2.2" + quiver: + dependency: transitive + description: + name: quiver + sha256: b1c1ac5ce6688d77f65f3375a9abb9319b3cb32486bdc7a1e0fdf004d7ba4e47 + url: "https://pub.dev" + source: hosted + version: "3.2.1" riverpod: dependency: transitive description: @@ -1211,6 +1227,22 @@ packages: url: "https://pub.dev" source: hosted version: "6.2.2" + xpath_selector: + dependency: transitive + description: + name: xpath_selector + sha256: "6d8295565a34a6e2821a0721592e584c70421e52a0f54955e0c8e41963db7c90" + url: "https://pub.dev" + source: hosted + version: "3.0.2" + xpath_selector_html_parser: + dependency: "direct main" + description: + name: xpath_selector_html_parser + sha256: ebc562a07832a4062a2bc1c238fd59a81d31d841f15460caeed412ce078af9e0 + url: "https://pub.dev" + source: hosted + version: "3.0.1" xxh3: dependency: transitive description: diff --git a/pubspec.yaml b/pubspec.yaml index 70b0511..ce54f68 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -60,6 +60,7 @@ dependencies: isar: 3.1.0+1 isar_flutter_libs: 3.1.0+1 share_plus: ^7.0.0 + xpath_selector_html_parser: ^3.0.1 # The following adds the Cupertino Icons font to your application.