swcs/ReadSharp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

invert HasNoHeadline; better NextPage parser;

Browse Source
This commit is contained in:
Tobias Klika
2014-02-17 22:10:20 +01:00
parent d754eda216
commit 3bbf4fdcb7
5 changed files with 58 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files
PortablePorts/NReadability/NReadabilityTranscoder.cs
+23 -2
View File
@@ -142,7 +142,9 @@ namespace ReadSharp.Ports.NReadability
{ new Regex("^https?://(www|blog)\\.bufferapp.com", RegexOptions.IgnoreCase), ".post" }, { new Regex("^https?://(www|blog)\\.bufferapp.com", RegexOptions.IgnoreCase), ".post" },
{ new Regex("^https?://(www.)?polygon.com", RegexOptions.IgnoreCase), ".body" }, { new Regex("^https?://(www.)?polygon.com", RegexOptions.IgnoreCase), ".body" },
{ new Regex("^https?://(www.)?gizmodo.com", RegexOptions.IgnoreCase), ".post-container" }, { new Regex("^https?://(www.)?gizmodo.com", RegexOptions.IgnoreCase), ".post-container" },
{ new Regex("^https?://(www.)?it-scoop.com", RegexOptions.IgnoreCase), ".entry-content" } { new Regex("^https?://(www.)?it-scoop.com", RegexOptions.IgnoreCase), ".entry-content" },
{ new Regex("^https?://(m\\.|www\\.)?youtube.com", RegexOptions.IgnoreCase), "#watch-description-content" },
{ new Regex("^https?://(www.)?vimeo.com", RegexOptions.IgnoreCase), ".description_wrapper" }
}; };
#endregion #endregion
@@ -545,10 +547,29 @@ namespace ReadSharp.Ports.NReadability
double urlComparity = StringCompare(url, linkHref); double urlComparity = StringCompare(url, linkHref);
/* If the next page differs from the original too much */ /* If the next page differs from the original too much */
if (StringCompare(url, linkHref) < 80) if (urlComparity < 80)
{ {
linkObj.Score -= 200; linkObj.Score -= 200;
} }
else if (urlComparity > 95 && url.Length <= linkHref.Length)
{
try
{
string originalLinkHrefNumber = Regex.Match(url.Replace(articleBaseUrl, ""), @"\d+").Value;
string linkHrefNumber = Regex.Match(linkHref.Replace(articleBaseUrl, ""), @"\d+").Value;
if (originalLinkHrefNumber == "")
{
originalLinkHrefNumber = "0";
}
if (originalLinkHrefNumber != null && linkHrefNumber != null && linkHrefNumber.Length < 3 && linkHref.EndsWith(linkHrefNumber) && Convert.ToByte(originalLinkHrefNumber) < Convert.ToByte(linkHrefNumber))
{
linkObj.Score += 75;
}
}
catch { }
}
/* If any ancestor node contains page or paging or paginat */ /* If any ancestor node contains page or paging or paginat */
XElement parentNode = linkElement.Parent; XElement parentNode = linkElement.Parent;
ReadSharp.Tests/ReadTests.cs
+21 -6
View File
@@ -22,7 +22,6 @@ namespace ReadSharp.Tests
Article result = await reader.Read(new Uri("http://frontendplay.com/story/4/http-caching-demystified-part-2-implementation")); Article result = await reader.Read(new Uri("http://frontendplay.com/story/4/http-caching-demystified-part-2-implementation"));
Assert.DoesNotContain("<!DOCTYPE html>", result.Content); Assert.DoesNotContain("<!DOCTYPE html>", result.Content);
Assert.Contains("<h1>", result.Content);
Assert.True(result.Content.Length > 15000); Assert.True(result.Content.Length > 15000);
} }
@@ -32,7 +31,6 @@ namespace ReadSharp.Tests
{ {
Article result = await reader.Read(new Uri("http://frontendplay.com/story/4/http-caching-demystified-part-2-implementation"), new ReadOptions() Article result = await reader.Read(new Uri("http://frontendplay.com/story/4/http-caching-demystified-part-2-implementation"), new ReadOptions()
{ {
HasNoHeadline = true,
HasHeaderTags = true HasHeaderTags = true
}); });
@@ -143,7 +141,10 @@ namespace ReadSharp.Tests
[Fact] [Fact]
public async Task AreMultipageArticlesWorking() public async Task AreMultipageArticlesWorking()
{ {
Article result = await reader.Read(new Uri("http://www.zeit.de/gesellschaft/2014-02/alice-schwarzer-steuerhinterziehung-doppelmoral")); Article result = await reader.Read(new Uri("http://www.anandtech.com/show/7594/samsung-ssd-840-evo-msata-120gb-250gb-500gb-1tb-review"));
Assert.Equal(result.NextPage.ToString(), "http://www.anandtech.com/show/7594/samsung-ssd-840-evo-msata-120gb-250gb-500gb-1tb-review/2");
result = await reader.Read(new Uri("http://www.zeit.de/gesellschaft/2014-02/alice-schwarzer-steuerhinterziehung-doppelmoral"));
Assert.Equal(result.NextPage.ToString(), "http://www.zeit.de/gesellschaft/2014-02/alice-schwarzer-steuerhinterziehung-doppelmoral/seite-2"); Assert.Equal(result.NextPage.ToString(), "http://www.zeit.de/gesellschaft/2014-02/alice-schwarzer-steuerhinterziehung-doppelmoral/seite-2");
result = await reader.Read(new Uri("http://www.sueddeutsche.de/wirtschaft/netzbetreiber-und-die-energiewende-im-kampf-gegen-blackouts-und-buergerproteste-1.1880754")); result = await reader.Read(new Uri("http://www.sueddeutsche.de/wirtschaft/netzbetreiber-und-die-energiewende-im-kampf-gegen-blackouts-und-buergerproteste-1.1880754"));
@@ -156,10 +157,14 @@ namespace ReadSharp.Tests
[Fact] [Fact]
public async Task AreSinglepageArticlesNotPopulatingNextPage() public async Task AreSinglepageArticlesNotPopulatingNextPage()
{ {
Article result = await reader.Read(new Uri("http://www.buzzfeed.com/mattlynley/the-16-most-interesting-things-to-come-out-of-bill-gates-qa")); Article result = await reader.Read(new Uri("http://arstechnica.com/apple/2014/01/two-steps-forward-a-review-of-the-2013-mac-pro/7/"));
Assert.Null(result.NextPage); Assert.Null(result.NextPage);
result = await reader.Read(new Uri("http://arstechnica.com/apple/2014/01/two-steps-forward-a-review-of-the-2013-mac-pro/7/")); result = await reader.Read(new Uri("http://www.wpcentral.com/developers-leak-new-features-windows-phone-81-sdk"), new ReadOptions() { MultipageDownload = true });
Assert.Null(result.NextPage);
Assert.Equal(result.PageCount, 1);
result = await reader.Read(new Uri("http://www.buzzfeed.com/mattlynley/the-16-most-interesting-things-to-come-out-of-bill-gates-qa"));
Assert.Null(result.NextPage); Assert.Null(result.NextPage);
result = await reader.Read(new Uri("http://www.sueddeutsche.de/wirtschaft/netzbetreiber-und-die-energiewende-im-kampf-gegen-blackouts-und-buergerproteste-1.1880754-2")); result = await reader.Read(new Uri("http://www.sueddeutsche.de/wirtschaft/netzbetreiber-und-die-energiewende-im-kampf-gegen-blackouts-und-buergerproteste-1.1880754-2"));
@@ -167,6 +172,7 @@ namespace ReadSharp.Tests
result = await reader.Read(new Uri("http://www.zeit.de/gesellschaft/2014-02/alice-schwarzer-steuerhinterziehung-doppelmoral/seite-2")); result = await reader.Read(new Uri("http://www.zeit.de/gesellschaft/2014-02/alice-schwarzer-steuerhinterziehung-doppelmoral/seite-2"));
Assert.Null(result.NextPage); Assert.Null(result.NextPage);
} }
[Fact] [Fact]
@@ -174,7 +180,10 @@ namespace ReadSharp.Tests
{ {
ReadOptions options = new ReadOptions() { MultipageDownload = true }; ReadOptions options = new ReadOptions() { MultipageDownload = true };
Article result = await reader.Read(new Uri("http://www.zeit.de/gesellschaft/2014-02/alice-schwarzer-steuerhinterziehung-doppelmoral"), options); Article result = await reader.Read(new Uri("http://www.anandtech.com/show/7594/samsung-ssd-840-evo-msata-120gb-250gb-500gb-1tb-review"), options);
Assert.Equal(result.PageCount, 9);
result = await reader.Read(new Uri("http://www.zeit.de/gesellschaft/2014-02/alice-schwarzer-steuerhinterziehung-doppelmoral"), options);
Assert.Equal(result.PageCount, 2); Assert.Equal(result.PageCount, 2);
Assert.True(result.WordCount > 800); Assert.True(result.WordCount > 800);
@@ -228,6 +237,12 @@ namespace ReadSharp.Tests
result = await reader.Read(new Uri("http://m.spiegel.de/spiegelgeschichte/a-946060.html")); result = await reader.Read(new Uri("http://m.spiegel.de/spiegelgeschichte/a-946060.html"));
Assert.DoesNotContain("Detecting browser settings", result.Content); Assert.DoesNotContain("Detecting browser settings", result.Content);
result = await reader.Read(new Uri("https://vimeo.com/84391640"));
Assert.Contains("twitter.com/pokiapp", result.Content);
result = await reader.Read(new Uri("http://www.youtube.com/watch?v=GI2lHSPkW1c"));
Assert.Contains("IT PAST MIDNIGHT A COUPLE HOURS AGO, IT'S FEELS COLDER", result.Content);
} }
ReadSharp/HtmlUtilities.cs
+3
View File
@@ -94,6 +94,9 @@ namespace ReadSharp
// treat paragraphs as crlf // treat paragraphs as crlf
outText.Write("\r\n"); outText.Write("\r\n");
break; break;
case "br":
outText.Write("\r\n");
break;
} }
if (node.HasChildNodes) if (node.HasChildNodes)
ReadSharp/Models/ReadOptions.cs
+2 -2
View File
@@ -17,7 +17,7 @@ namespace ReadSharp
/// <value> /// <value>
/// <c>true</c> if [has no headline]; otherwise, <c>false</c>. /// <c>true</c> if [has no headline]; otherwise, <c>false</c>.
/// </value> /// </value>
public bool HasNoHeadline { get; set; } public bool HasHeadline { get; set; }
/// <summary> /// <summary>
/// Are deep links with hashes not transformed to absolute URIs (default: false). /// Are deep links with hashes not transformed to absolute URIs (default: false).
@@ -52,7 +52,7 @@ namespace ReadSharp
return new ReadOptions() return new ReadOptions()
{ {
HasHeaderTags = false, HasHeaderTags = false,
HasNoHeadline = false, HasHeadline = false,
UseDeepLinks = false, UseDeepLinks = false,
PrettyPrint = false, PrettyPrint = false,
MultipageDownload = false MultipageDownload = false
ReadSharp/Reader.cs
+9 -2
View File
@@ -214,7 +214,7 @@ namespace ReadSharp
DomSerializationParams = new DomSerializationParams() DomSerializationParams = new DomSerializationParams()
{ {
BodyOnly = !options.HasHeaderTags, BodyOnly = !options.HasHeaderTags,
NoHeadline = options.HasNoHeadline, NoHeadline = !options.HasHeadline,
PrettyPrint = options.PrettyPrint, PrettyPrint = options.PrettyPrint,
DontIncludeContentTypeMetaElement = true, DontIncludeContentTypeMetaElement = true,
DontIncludeMobileSpecificMetaElements = true, DontIncludeMobileSpecificMetaElements = true,
@@ -352,6 +352,14 @@ namespace ReadSharp
Encoding = encoding Encoding = encoding
}; };
// in same special cases their are multiple pages, which are only comments or do not contain new content.
// if this is the case we will break here and return the first page only.
if (previousResponse != null && previousResponse.TranscodingResult.ExtractedContent == transcodingResult.ExtractedContent)
{
previousResponse.TranscodingResult.NextPageUrl = null;
return previousResponse;
}
// multiple pages are available // multiple pages are available
try try
{ {
@@ -359,7 +367,6 @@ namespace ReadSharp
{ {
return await Request(new Uri(transcodingResult.NextPageUrl), new ReadOptions() return await Request(new Uri(transcodingResult.NextPageUrl), new ReadOptions()
{ {
HasNoHeadline = true,
PrettyPrint = options.PrettyPrint, PrettyPrint = options.PrettyPrint,
UseDeepLinks = options.UseDeepLinks, UseDeepLinks = options.UseDeepLinks,
MultipageDownload = true MultipageDownload = true