diff --git a/PortablePorts/NReadability/NReadabilityTranscoder.cs b/PortablePorts/NReadability/NReadabilityTranscoder.cs index a517afe..c982c3b 100644 --- a/PortablePorts/NReadability/NReadabilityTranscoder.cs +++ b/PortablePorts/NReadability/NReadabilityTranscoder.cs @@ -106,6 +106,7 @@ namespace ReadSharp.Ports.NReadability private static readonly Regex _PositiveWeightRegex = new Regex("article|body|content|entry|hentry|main|page|pagination|post|text|blog|story|paragraph|instapaper_body|entry-content|article-body|entry-body", RegexOptions.IgnoreCase); private static readonly Regex _NegativeWeightRegex = new Regex("combx|comment|com-|contact|foot|footer|footnote|masthead|media|meta|outbrain|promo|related|scroll|shoutbox|sidebar|side|sponsor|shopping|tags|tool|widget", RegexOptions.IgnoreCase); private static readonly Regex _NegativeLinkParentRegex = new Regex("(stories|articles|news|documents|posts|notes|series|historie|artykuly|artykuły|wpisy|dokumenty|serie|geschichten|erzählungen|erzahlungen)", RegexOptions.IgnoreCase); + private static readonly Regex _PositiveNavPaginationRegex = new Regex("pagination|pages|numbers|seite|page", RegexOptions.IgnoreCase); private static readonly Regex _Extraneous = new Regex("print|archive|comment|discuss|e[-]?mail|share|reply|all|login|sign|single|also", RegexOptions.IgnoreCase); private static readonly Regex _DivToPElementsRegex = new Regex("<(a|blockquote|dl|div|img|ol|p|pre|table|ul)", RegexOptions.IgnoreCase); private static readonly Regex _EndOfSentenceRegex = new Regex("\\.( |$)", RegexOptions.Multiline); @@ -120,7 +121,7 @@ namespace ReadSharp.Ports.NReadability private static readonly Regex _ArticleTitleDashRegex3 = new Regex("[^\\|\\-]*[\\|\\-](.*)"); private static readonly Regex _ArticleTitleColonRegex1 = new Regex(".*:(.*)"); private static readonly Regex _ArticleTitleColonRegex2 = new Regex("[^:]*[:](.*)"); - private static readonly Regex _NextLink = new Regex(@"(next|weiter|continue|dalej|następna|nastepna>([^\|]|$)|�([^\|]|$))", RegexOptions.IgnoreCase); + private static readonly Regex _NextLink = new Regex(@"(next|weiter|nächste|nächstes|^([0-9]\.)|continue|dalej|następna|nastepna>([^\|]|$)|�([^\|]|$))", RegexOptions.IgnoreCase); private static readonly Regex _NextStoryLink = new Regex("(story|article|news|document|post|note|series|historia|artykul|artykuł|wpis|dokument|seria|geschichte|erzählung|erzahlung|artikel|serie)", RegexOptions.IgnoreCase); private static readonly Regex _PrevLink = new Regex("(prev|earl|[^b]old|new|wstecz|poprzednia|<|�)", RegexOptions.IgnoreCase); private static readonly Regex _PageRegex = new Regex("pag(e|ing|inat)|([^a-z]|^)pag([^a-z]|$)", RegexOptions.IgnoreCase); @@ -782,7 +783,11 @@ namespace ReadSharp.Ports.NReadability /* Remove all nav tags. */ elementsToRemove.Clear(); - elementsToRemove.AddRange(rootElement.GetElementsByTagName("nav")); + elementsToRemove.AddRange(rootElement.GetElementsByTagName("nav").Where(item => + { + string classList = item.GetClass(); + return String.IsNullOrEmpty(classList) || (classList != null && !_PositiveNavPaginationRegex.IsMatch(classList)); + })); RemoveElements(elementsToRemove); /* Remove all anchors. */ diff --git a/ReadSharp.Tests/ReadTests.cs b/ReadSharp.Tests/ReadTests.cs index d75b859..e8d42bc 100644 --- a/ReadSharp.Tests/ReadTests.cs +++ b/ReadSharp.Tests/ReadTests.cs @@ -162,6 +162,9 @@ namespace ReadSharp.Tests result = await reader.Read(new Uri("http://www.polygon.com/2014/1/31/5364728/super-bowl-xlviii-xbox-activities-new-york")); Assert.True(result.Content.Contains("week for Super Bowl XLVIII") && result.Content.Contains("two tickets to the Super Bowl.")); + + result = await reader.Read(new Uri("http://habrahabr.ru/post/211905/")); + Assert.NotEmpty(result.Content); } [Fact] @@ -178,11 +181,17 @@ namespace ReadSharp.Tests [Fact] public async Task AreMultipageArticlesWorking() { - Article result = await reader.Read(new Uri("http://www.pcwelt.de/ratgeber/Acht_Tipps_fuer_die_sichere_Cloud-Google_Drive__Microsoft_Skydrive__Teamdrive-8205869.html?redirect=1")); - Assert.NotNull(result.NextPage); + Article result = await reader.Read(new Uri("http://www.zeit.de/gesellschaft/2014-02/alice-schwarzer-steuerhinterziehung-doppelmoral")); + Assert.Equal(result.NextPage.ToString(), "http://www.zeit.de/gesellschaft/2014-02/alice-schwarzer-steuerhinterziehung-doppelmoral/seite-2"); + + result = await reader.Read(new Uri("http://www.sueddeutsche.de/wirtschaft/netzbetreiber-und-die-energiewende-im-kampf-gegen-blackouts-und-buergerproteste-1.1880754")); + Assert.Equal(result.NextPage.ToString(), "http://www.sueddeutsche.de/wirtschaft/netzbetreiber-und-die-energiewende-im-kampf-gegen-blackouts-und-buergerproteste-1.1880754-2"); + + result = await reader.Read(new Uri("http://www.pcwelt.de/ratgeber/Acht_Tipps_fuer_die_sichere_Cloud-Google_Drive__Microsoft_Skydrive__Teamdrive-8205869.html?redirect=1")); + Assert.Equal(result.NextPage.ToString(), "http://www.pcwelt.de/ratgeber/Google_Drive__Microsoft_Skydrive__Teamdrive-Tipp_3_-_Server-Standort_beachten-8205887.html"); result = await reader.Read(new Uri("http://arstechnica.com/apple/2014/01/two-steps-forward-a-review-of-the-2013-mac-pro/")); - Assert.NotNull(result.NextPage); + Assert.Equal(result.NextPage.ToString(), "http://arstechnica.com/apple/2014/01/two-steps-forward-a-review-of-the-2013-mac-pro/2"); } [Fact]