swcs/ReadSharp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

update next page URL parser

Browse Source
This commit is contained in:
Tobias Klika
2014-02-13 00:13:47 +01:00
parent 4f01f25a72
commit ac3b02faf4
2 changed files with 19 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
PortablePorts/NReadability/NReadabilityTranscoder.cs
+7 -2
View File
@@ -106,6 +106,7 @@ namespace ReadSharp.Ports.NReadability
private static readonly Regex _PositiveWeightRegex = new Regex("article|body|content|entry|hentry|main|page|pagination|post|text|blog|story|paragraph|instapaper_body|entry-content|article-body|entry-body", RegexOptions.IgnoreCase);
private static readonly Regex _NegativeWeightRegex = new Regex("combx|comment|com-|contact|foot|footer|footnote|masthead|media|meta|outbrain|promo|related|scroll|shoutbox|sidebar|side|sponsor|shopping|tags|tool|widget", RegexOptions.IgnoreCase);
private static readonly Regex _NegativeLinkParentRegex = new Regex("(stories|articles|news|documents|posts|notes|series|historie|artykuly|artykuły|wpisy|dokumenty|serie|geschichten|erzählungen|erzahlungen)", RegexOptions.IgnoreCase);
private static readonly Regex _PositiveNavPaginationRegex = new Regex("pagination|pages|numbers|seite|page", RegexOptions.IgnoreCase);
private static readonly Regex _Extraneous = new Regex("print|archive|comment|discuss|e[-]?mail|share|reply|all|login|sign|single|also", RegexOptions.IgnoreCase);
private static readonly Regex _DivToPElementsRegex = new Regex("<(a|blockquote|dl|div|img|ol|p|pre|table|ul)", RegexOptions.IgnoreCase);
private static readonly Regex _EndOfSentenceRegex = new Regex("\\.( |$)", RegexOptions.Multiline);
@@ -120,7 +121,7 @@ namespace ReadSharp.Ports.NReadability
private static readonly Regex _ArticleTitleDashRegex3 = new Regex("[^\\|\\-]*[\\|\\-](.*)");
private static readonly Regex _ArticleTitleColonRegex1 = new Regex(".*:(.*)");
private static readonly Regex _ArticleTitleColonRegex2 = new Regex("[^:]*[:](.*)");
private static readonly Regex _NextLink = new Regex(@"(next|weiter|continue|dalej|następna|nastepna>([^\|]|$)|�([^\|]|$))", RegexOptions.IgnoreCase);
private static readonly Regex _NextLink = new Regex(@"(next|weiter|nächste|nächstes|^([0-9]\.)|continue|dalej|następna|nastepna>([^\|]|$)|�([^\|]|$))", RegexOptions.IgnoreCase);
private static readonly Regex _NextStoryLink = new Regex("(story|article|news|document|post|note|series|historia|artykul|artykuł|wpis|dokument|seria|geschichte|erzählung|erzahlung|artikel|serie)", RegexOptions.IgnoreCase);
private static readonly Regex _PrevLink = new Regex("(prev|earl|[^b]old|new|wstecz|poprzednia|<|�)", RegexOptions.IgnoreCase);
private static readonly Regex _PageRegex = new Regex("pag(e|ing|inat)|([^a-z]|^)pag([^a-z]|$)", RegexOptions.IgnoreCase);
@@ -782,7 +783,11 @@ namespace ReadSharp.Ports.NReadability
/* Remove all nav tags. */
elementsToRemove.Clear();
elementsToRemove.AddRange(rootElement.GetElementsByTagName("nav"));
elementsToRemove.AddRange(rootElement.GetElementsByTagName("nav").Where(item =>
{
string classList = item.GetClass();
return String.IsNullOrEmpty(classList) || (classList != null && !_PositiveNavPaginationRegex.IsMatch(classList));
}));
RemoveElements(elementsToRemove);
/* Remove all anchors. */
ReadSharp.Tests/ReadTests.cs
+12 -3
View File
@@ -162,6 +162,9 @@ namespace ReadSharp.Tests
result = await reader.Read(new Uri("http://www.polygon.com/2014/1/31/5364728/super-bowl-xlviii-xbox-activities-new-york"));
Assert.True(result.Content.Contains("week for Super Bowl XLVIII") && result.Content.Contains("two tickets to the Super Bowl."));
result = await reader.Read(new Uri("http://habrahabr.ru/post/211905/"));
Assert.NotEmpty(result.Content);
}
[Fact]
@@ -178,11 +181,17 @@ namespace ReadSharp.Tests
[Fact]
public async Task AreMultipageArticlesWorking()
{
Article result = await reader.Read(new Uri("http://www.pcwelt.de/ratgeber/Acht_Tipps_fuer_die_sichere_Cloud-Google_Drive__Microsoft_Skydrive__Teamdrive-8205869.html?redirect=1"));
Assert.NotNull(result.NextPage);
Article result = await reader.Read(new Uri("http://www.zeit.de/gesellschaft/2014-02/alice-schwarzer-steuerhinterziehung-doppelmoral"));
Assert.Equal(result.NextPage.ToString(), "http://www.zeit.de/gesellschaft/2014-02/alice-schwarzer-steuerhinterziehung-doppelmoral/seite-2");
result = await reader.Read(new Uri("http://www.sueddeutsche.de/wirtschaft/netzbetreiber-und-die-energiewende-im-kampf-gegen-blackouts-und-buergerproteste-1.1880754"));
Assert.Equal(result.NextPage.ToString(), "http://www.sueddeutsche.de/wirtschaft/netzbetreiber-und-die-energiewende-im-kampf-gegen-blackouts-und-buergerproteste-1.1880754-2");
result = await reader.Read(new Uri("http://www.pcwelt.de/ratgeber/Acht_Tipps_fuer_die_sichere_Cloud-Google_Drive__Microsoft_Skydrive__Teamdrive-8205869.html?redirect=1"));
Assert.Equal(result.NextPage.ToString(), "http://www.pcwelt.de/ratgeber/Google_Drive__Microsoft_Skydrive__Teamdrive-Tipp_3_-_Server-Standort_beachten-8205887.html");
result = await reader.Read(new Uri("http://arstechnica.com/apple/2014/01/two-steps-forward-a-review-of-the-2013-mac-pro/"));
Assert.NotNull(result.NextPage);
Assert.Equal(result.NextPage.ToString(), "http://arstechnica.com/apple/2014/01/two-steps-forward-a-review-of-the-2013-mac-pro/2");
}
[Fact]