Updated lots of stuff, got multi scrape working, need to test not-nullable chapter novel ids with our current model, now supports sqlite and postgres concurrently (and easy add more), need to get it deployed/do auth

WebNovelPortalAPI/Scrapers/SyosetuScraper.cs

@@ -1,10 +1,15 @@
+using System.Text.RegularExpressions;
+using HtmlAgilityPack;
+using Treestar.Shared.Models.DBDomain;
+
 namespace WebNovelPortalAPI.Scrapers;
 
 public class SyosetuScraper : AbstractScraper
 {
+
     protected override string UrlMatchPattern => @"https?:\/\/\w+\.syosetu\.com\/\w+\/?";
 
-    protected override string BaseUrlPattern => @"https?:\/\/\w+\.syosetu\.com\/?";
+    protected override string BaseUrlPattern => @"https?:\/\/\w+\.syosetu\.com";
 
     protected override string? WorkTitlePattern => @"//p[@class='novel_title']";
 
@@ -14,15 +19,101 @@ public class SyosetuScraper : AbstractScraper
 
     protected override string? ChapterUrlPattern => @"//dl[@class='novel_sublist2']//a";
 
-    protected override string? ChapterNamePattern => @"//dl[@class='novel_sublist2']//a";
+    protected override string? ChapterPostedPattern => @"following-sibling::dt[@class='long_update']";
 
-    protected override string? ChapterPostedPattern => base.ChapterPostedPattern;
+    protected override string? ChapterUpdatedPattern => @"span";
 
-    protected override string? ChapterUpdatedPattern => base.ChapterUpdatedPattern;
+    protected override string? TagPattern => @"//th[text()='キーワード']/following-sibling::td";
 
-    protected override string? TagPattern => base.TagPattern;
+    protected override string? DatePostedPattern => @"//th[text()='掲載日']/following-sibling::td";
 
-    protected override string? DatePostedPattern => base.DatePostedPattern;
+    protected override string? DateUpdatedPattern => @"//th[contains(text(),'掲載日')]/following-sibling::td";
 
-    protected override string? DateUpdatedPattern => base.DateUpdatedPattern;
+    private HtmlDocument? GetInfoPage(string baseUrl, string novelUrl)
+    {
+        string novelInfoBase = $"/novelview/infotop/ncode/";
+        string novelRegex = @"https?:\/\/\w+\.syosetu\.com\/(\w+)\/?";
+        string novelCode = new Regex(novelRegex).Match(novelUrl).Groups[1].Value;
+        string novelInfoPage = $"{baseUrl}{novelInfoBase}{novelCode}";
+        var web = new HtmlWeb();
+        return web.Load(novelInfoPage);
+    }
+
+    protected override List<Chapter> GetChapters(HtmlDocument document, string baseUrl, string novelUrl)
+    {
+        string dateUpdatedRegex = @"\d\d\d\d\/\d\d\/\d\d \d\d:\d\d";
+        var nodes = document.DocumentNode.SelectNodes(ChapterUrlPattern);
+        return nodes.Select((node,i) =>
+        {
+            var datePostedNode = node.ParentNode.SelectSingleNode(ChapterPostedPattern);
+            var datePosted = DateTime.Parse(new Regex(dateUpdatedRegex).Match(datePostedNode.InnerText).Value);
+            var dateUpdatedNode = datePostedNode.SelectSingleNode(ChapterUpdatedPattern);
+            DateTime dateUpdated;
+            if (dateUpdatedNode == null)
+            {
+                dateUpdated = datePosted;
+            }
+            else
+            {
+                dateUpdated = DateTime.Parse(new Regex(dateUpdatedRegex).Match(dateUpdatedNode.Attributes["title"].Value).Value);
+            }
+            return new Chapter
+            {
+                Name = node.InnerText,
+                Url = baseUrl + node.Attributes["href"].Value,
+                ChapterNumber = i+1,
+                DatePosted = datePosted,
+                DateUpdated = dateUpdated
+            };
+        }).ToList();
+    }
+
+    protected override Author GetAuthor(HtmlDocument document, string baseUrl, string novelUrl)
+    {
+        var authorLink = document.DocumentNode.SelectSingleNode(AuthorLinkPattern)?.Attributes["href"].Value ?? null;
+        if (string.IsNullOrEmpty(authorLink))
+        {
+            return null;
+        }
+        var authorName = document.DocumentNode.SelectSingleNode(AuthorNamePattern).InnerText.Replace("\n", "");
+        return new Author
+        {
+            Name = authorName,
+            Url = authorLink
+        };
+    }
+
+    protected override DateTime GetPostedDate(HtmlDocument document, string baseUrl, string novelUrl)
+    {
+        var doc = GetInfoPage(baseUrl, novelUrl);
+        if (doc == null)
+        {
+            return DateTime.MinValue;
+        }
+
+        var node = doc.DocumentNode.SelectSingleNode(DatePostedPattern);
+        return DateTime.Parse(node.InnerText);
+    }
+
+    protected override DateTime GetLastUpdatedDate(HtmlDocument document, string baseUrl, string novelUrl)
+    {
+        var doc = GetInfoPage(baseUrl, novelUrl);
+        if (doc == null)
+        {
+            return DateTime.MinValue;
+        }
+        return DateTime.Parse(doc.DocumentNode.SelectNodes(DateUpdatedPattern)[1].InnerText);
+    }
+
+    protected override List<Tag> GetTags(HtmlDocument document, string baseUrl, string novelUrl)
+    {
+        var doc = GetInfoPage(baseUrl, novelUrl);
+        if (doc == null)
+        {
+            return new List<Tag>();
+        }
+
+        var tags = doc.DocumentNode.SelectSingleNode(TagPattern).InnerText.Replace("\n", "").Replace("&nbsp;", " ").Split(' ');
+        return tags.Select(i => new Tag {TagValue = i}).ToList();
+    }
 }