Updated lots of stuff, got multi scrape working, need to test not-nullable chapter novel ids with our current model, now supports sqlite and postgres concurrently (and easy add more), need to get it deployed/do auth
Some checks failed
continuous-integration/drone/push Build is failing

This commit is contained in:
2022-07-16 17:17:43 -04:00
parent eab3399268
commit d98324c11e
73 changed files with 1591 additions and 680 deletions

View File

@@ -1,6 +1,5 @@
using System.Reflection.Metadata;
using System.Text.RegularExpressions;
using DBConnection.Models;
using HtmlAgilityPack;
namespace WebNovelPortalAPI.Scrapers;
@@ -19,7 +18,7 @@ public class KakuyomuScraper : AbstractScraper
protected override string? ChapterNamePattern => @"span";
protected override string? ChapterPostedPattern => base.ChapterPostedPattern;
protected override string? ChapterPostedPattern => @"time";
protected override string? ChapterUpdatedPattern => base.ChapterUpdatedPattern;
@@ -29,8 +28,10 @@ public class KakuyomuScraper : AbstractScraper
protected override string? DateUpdatedPattern => @"//time[@itemprop='dateModified']";
public string? ScrapeChapterContent(string chapterUrl)
protected override (DateTime? Posted, DateTime? Updated) GetDateTimeForChapter(HtmlNode linkNode, HtmlNode baseNode, string baseUrl,
string novelUrl)
{
throw new NotImplementedException();
var datePosted = linkNode.SelectSingleNode(ChapterPostedPattern).Attributes["datetime"].Value;
return (DateTime.Parse(datePosted), null);
}
}