swcs/ReadSharp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

respect for containers surrounding images

Browse Source
This commit is contained in:
Tobias Klika
2014-01-30 21:57:37 +01:00
parent 122ccfe7b6
commit eb053f6df4
2 changed files with 23 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files
PortablePorts/NReadability/NReadabilityTranscoder.cs
+13 -11
View File
@@ -124,7 +124,8 @@ namespace ReadSharp.Ports.NReadability
private static readonly Regex _NextStoryLink = new Regex("(story|article|news|document|post|note|series|historia|artykul|artykuł|wpis|dokument|seria|geschichte|erzählung|erzahlung|artikel|serie)", RegexOptions.IgnoreCase);
private static readonly Regex _PrevLink = new Regex("(prev|earl|[^b]old|new|wstecz|poprzednia|<|�)", RegexOptions.IgnoreCase);
private static readonly Regex _PageRegex = new Regex("pag(e|ing|inat)|([^a-z]|^)pag([^a-z]|$)", RegexOptions.IgnoreCase);
private static readonly Regex _LikelyParagraphDivRegex = new Regex("text|para|parbase|paragraph", RegexOptions.IgnoreCase);
private static readonly Regex _LikelyParagraphDivRegex = new Regex("text|para|parbase|paragraph|figure", RegexOptions.IgnoreCase);
private static readonly Regex _LikelyImageContainerRegex = new Regex("image|figure|photo|media", RegexOptions.IgnoreCase);
#endregion
@@ -751,14 +752,9 @@ namespace ReadSharp.Ports.NReadability
rootElement.GetElementsByTagName("script")
.ForEach(scriptElement =>
{
string scriptSrc = scriptElement.GetAttributeValue("src", null);
if (string.IsNullOrEmpty(scriptSrc) || scriptSrc.LastIndexOf("readability") == -1)
{
elementsToRemove.Add(scriptElement);
}
});
{
elementsToRemove.Add(scriptElement);
});
RemoveElements(elementsToRemove);
@@ -1260,6 +1256,7 @@ namespace ReadSharp.Ports.NReadability
/* Clean out junk from the article content. */
Clean(articleContentElement, "form");
Clean(articleContentElement, "object");
Clean(articleContentElement, "meta");
if (articleContentElement.GetElementsByTagName("h1").Count() == 1)
{
@@ -1279,7 +1276,7 @@ namespace ReadSharp.Ports.NReadability
/* Do these last as the previous stuff may have removed junk that will affect these. */
CleanConditionally(articleContentElement, "table");
CleanConditionally(articleContentElement, "ul");
CleanConditionally(articleContentElement, "div");
CleanConditionally(articleContentElement, "div", true);
/* Remove extra paragraphs. */
IEnumerable<XElement> paraElements = articleContentElement.GetElementsByTagName("p");
@@ -1460,7 +1457,7 @@ namespace ReadSharp.Ports.NReadability
/// Cleans a <paramref name="rootElement" /> of all elements with name <paramref name="elementName" /> if they look fishy.
/// "Fishy" is an algorithm based on content length, classnames, link density, number of images and embeds, etc.
/// </summary>
internal void CleanConditionally(XElement rootElement, string elementName)
internal void CleanConditionally(XElement rootElement, string elementName, bool checkImageContainer = false)
{
if (elementName == null)
{
@@ -1498,6 +1495,11 @@ namespace ReadSharp.Ports.NReadability
int lisCount = element.GetElementsByTagName("li").Count();
int inputsCount = element.GetElementsByTagName("input").Count();
if (checkImageContainer && imgsCount > 0 && _LikelyImageContainerRegex.IsMatch(element.GetClass()))
{
continue;
}
// while counting embeds we omit video-embeds
int embedsCount =
element.GetElementsByTagName("embed")
ReadSharp.Tests/ReadTests.cs
+10 -6
View File
@@ -1,4 +1,5 @@
using System;
using System.Linq;
using System.Threading.Tasks;
using Xunit;
@@ -137,14 +138,17 @@ namespace ReadSharp.Tests
[Fact]
public async Task TestCriticalURIs()
{
Article result = await reader.Read(new Uri("http://wpcentral.com.feedsportal.com/c/33999/f/616880/s/35a02b5e/sc/15/l/0L0Swpcentral0N0Cgameloft0Ediscusses0Etheir0Enew0Egame0Ebrothers0Earms0E30Esons0Ewar0Eceslive/story01.htm"));
Assert.NotEmpty(result.Content);
//Article result = await reader.Read(new Uri("http://wpcentral.com.feedsportal.com/c/33999/f/616880/s/35a02b5e/sc/15/l/0L0Swpcentral0N0Cgameloft0Ediscusses0Etheir0Enew0Egame0Ebrothers0Earms0E30Esons0Ewar0Eceslive/story01.htm"));
//Assert.NotEmpty(result.Content);
result = await reader.Read(new Uri("http://msdn.microsoft.com/en-us/library/windows/apps/hh464925.aspx"));
Assert.NotEmpty(result.Content);
//result = await reader.Read(new Uri("http://msdn.microsoft.com/en-us/library/windows/apps/hh464925.aspx"));
//Assert.NotEmpty(result.Content);
result = await reader.Read(new Uri("http://bit.ly/KAh7FJ"));
Assert.NotEmpty(result.Content);
//result = await reader.Read(new Uri("http://bit.ly/KAh7FJ"));
//Assert.NotEmpty(result.Content);
Article result = await reader.Read(new Uri("http://www.nytimes.com/2014/01/31/world/europe/ukraine-unrest.html?hp&_r=0"));
Assert.True(result.Images != null && result.Images.Count > 0 && result.Images.SingleOrDefault(item => item.Uri.OriginalString == "http://static01.nyt.com/images/2014/01/31/world/31ukraine-cnd01/31ukraine-cnd01-articleLarge.jpg") != null);
}
[Fact]