private void GetPagesRecursive(HtmlAgilityPack.HtmlNode document, IDictionary<int, string> pages)
{
var chapterPages = document.SelectNodes(@"//div[@class=""main-body""]//div[@class=""btn-group""][2]/ul[@class=""dropdown-menu""]/li/a");
if (chapterPages == null)
{
throw new ParserException("Could not find expected elements on website.", document.InnerHtml);
}
int addedCount = 0;
foreach (var pageLink in chapterPages)
{
int pageNumber = 0;
var url = GetFullUrl(pageLink.Attributes["href"].Value);
if (pages.Any(kvp => kvp.Value == url)) // skip duplicate urls
continue;
if (!Int32.TryParse(Regex.Match(pageLink.InnerText, @"\d+").Value, out pageNumber))
_log.Error("Unable to parse page number '" + pageLink.InnerText + "'");
if (pages.ContainsKey(pageNumber)) // if page is already in dictionary use random number instead
pageNumber = Random;
pages.Add(pageNumber, url);
addedCount++;
}
if (addedCount > 0)
{
var pageRecord = pages.OrderByDescending(kvp => kvp.Key).Skip(1).FirstOrDefault();
if (pageRecord.Equals(default(KeyValuePair<int, string>)))
return;
var nextDocument = WebHelper.GetHtmlDocument(pageRecord.Value);
GetPagesRecursive(nextDocument, pages);
}
}