swcs/ReadSharp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

give div's in article content the same weight as p's

Browse Source
This commit is contained in:
Tobias Klika
2014-02-16 13:52:12 +01:00
parent 9cc55170b7
commit b6c610ec85
2 changed files with 49 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files
PortablePorts/NReadability/NReadabilityTranscoder.cs
+8 -2
View File
@@ -1300,12 +1300,18 @@ namespace ReadSharp.Ports.NReadability
/* Do these last as the previous stuff may have removed junk that will affect these. */
CleanConditionally(articleContentElement, "table");
CleanConditionally(articleContentElement, "ul");
CleanConditionally(articleContentElement, "div", true);
//CleanConditionally(articleContentElement, "div", true);
/* Remove extra paragraphs. */
IEnumerable<XElement> paraElements = articleContentElement.GetElementsByTagName("p");
List<XElement> paraElements = articleContentElement.GetElementsByTagName("p").ToList();
IEnumerable<XElement> divElements = articleContentElement.GetElementsByTagName("div");
var elementsToRemove = new List<XElement>();
if (divElements != null)
{
paraElements.AddRange(divElements);
}
foreach (XElement paraElement in paraElements)
{
string innerText = GetInnerText(paraElement, false);
ReadSharp.Tests/ReadTests.cs
+41 -40
View File
@@ -99,16 +99,6 @@ namespace ReadSharp.Tests
}
[Fact]
public async Task ArePropertiesCorrectlyAssignedWithEmptyArticle()
{
Article result = await reader.Read(new Uri("https://docs.google.com/presentation/d/1n4NyG4uPRjAA8zn_pSQ_Ket0RhcWC6QlZ6LMjKeECo0/preview?sle=true#slide=id.g178014302_016"));
Assert.False(result.ContentExtracted);
Assert.True(result.WordCount == 7);
Assert.Equal(result.Title, result.PlainContent);
}
[Fact]
public async Task TestCzechCharsets()
{
@@ -139,34 +129,6 @@ namespace ReadSharp.Tests
Assert.NotEmpty(result.Content);
}
[Fact]
public async Task TestCriticalURIs()
{
Article result = await reader.Read(new Uri("http://wpcentral.com.feedsportal.com/c/33999/f/616880/s/35a02b5e/sc/15/l/0L0Swpcentral0N0Cgameloft0Ediscusses0Etheir0Enew0Egame0Ebrothers0Earms0E30Esons0Ewar0Eceslive/story01.htm"));
Assert.NotEmpty(result.Content);
result = await reader.Read(new Uri("http://www.fastcoexist.com/3016005/futurist-forum/10-creative-ideas-for-thriving-cities-of-the-future"));
Assert.Contains("1: 311", result.Content);
result = await reader.Read(new Uri("http://msdn.microsoft.com/en-us/library/windows/apps/hh464925.aspx"));
Assert.NotEmpty(result.Content);
result = await reader.Read(new Uri("http://bit.ly/KAh7FJ"));
Assert.NotEmpty(result.Content);
result = await reader.Read(new Uri("http://www.nytimes.com/2014/01/31/world/europe/ukraine-unrest.html?hp&_r=0"));
Assert.True(result.Images != null && result.Images.Count > 0);
result = await reader.Read(new Uri("http://www.polygon.com/2013/2/25/4026668/tomb-raider-review"));
Assert.True(result.Images != null && result.Images.Count > 3 && result.Content.Contains("For a reboot of a series that had lost its focus and purpose"));
result = await reader.Read(new Uri("http://www.polygon.com/2014/1/31/5364728/super-bowl-xlviii-xbox-activities-new-york"));
Assert.True(result.Content.Contains("week for Super Bowl XLVIII") && result.Content.Contains("two tickets to the Super Bowl."));
result = await reader.Read(new Uri("http://habrahabr.ru/post/211905/"));
Assert.NotEmpty(result.Content);
}
[Fact]
public async Task TestHintUrlsReturnFullArticles()
{
@@ -194,13 +156,52 @@ namespace ReadSharp.Tests
Assert.Equal(result.NextPage.ToString(), "http://arstechnica.com/apple/2014/01/two-steps-forward-a-review-of-the-2013-mac-pro/2");
}
[Fact]
public async Task TestCriticalURIs()
{
Article result = await reader.Read(new Uri("http://wpcentral.com.feedsportal.com/c/33999/f/616880/s/35a02b5e/sc/15/l/0L0Swpcentral0N0Cgameloft0Ediscusses0Etheir0Enew0Egame0Ebrothers0Earms0E30Esons0Ewar0Eceslive/story01.htm"));
Assert.NotEmpty(result.Content);
result = await reader.Read(new Uri("http://www.fastcoexist.com/3016005/futurist-forum/10-creative-ideas-for-thriving-cities-of-the-future"));
Assert.Contains("1: 311", result.Content);
result = await reader.Read(new Uri("http://msdn.microsoft.com/en-us/library/windows/apps/hh464925.aspx"));
Assert.NotEmpty(result.Content);
result = await reader.Read(new Uri("http://bit.ly/KAh7FJ"));
Assert.NotEmpty(result.Content);
result = await reader.Read(new Uri("http://www.nytimes.com/2014/01/31/world/europe/ukraine-unrest.html?hp&_r=0"));
Assert.True(result.Images != null && result.Images.Count > 0);
result = await reader.Read(new Uri("http://www.polygon.com/2013/2/25/4026668/tomb-raider-review"));
Assert.True(result.Images != null && result.Images.Count > 3 && result.Content.Contains("For a reboot of a series that had lost its focus and purpose"));
result = await reader.Read(new Uri("http://www.polygon.com/2014/1/31/5364728/super-bowl-xlviii-xbox-activities-new-york"));
Assert.True(result.Content.Contains("week for Super Bowl XLVIII") && result.Content.Contains("two tickets to the Super Bowl."));
result = await reader.Read(new Uri("http://habrahabr.ru/post/211905/"));
Assert.NotEmpty(result.Content);
result = await reader.Read(new Uri("http://www.dgtle.com/article-5682-1.html"));
Assert.Contains("http://img.dgtle.com/forum/201402/13/162237x8oumb8i0i0y0087.jpeg!680px", result.Content);
}
[Fact]
public async Task TestCriticalURIs2()
{
Article result = await reader.Read(new Uri("http://www.dgtle.com/article-5682-1.html"));
Assert.Contains("http://img.dgtle.com/forum/201402/13/162237x8oumb8i0i0y0087.jpeg!680px", result.Content);
}
[Fact]
public async Task DebugArticle()
{
string uri = "http://habrahabr.ru/post/211905/";
string uri = "http://www.dgtle.com/article-5682-1.html";
Article result = await reader.Read(new Uri(uri));
Assert.NotEmpty(result.Content);
Assert.Contains("http://img.dgtle.com/forum/201402/13/162237x8oumb8i0i0y0087.jpeg!680px", result.Content);
}
}
}