diff --git a/PortablePorts/NReadability/NReadabilityTranscoder.cs b/PortablePorts/NReadability/NReadabilityTranscoder.cs index d415ec1..bb58c0a 100644 --- a/PortablePorts/NReadability/NReadabilityTranscoder.cs +++ b/PortablePorts/NReadability/NReadabilityTranscoder.cs @@ -124,7 +124,8 @@ namespace ReadSharp.Ports.NReadability private static readonly Regex _NextStoryLink = new Regex("(story|article|news|document|post|note|series|historia|artykul|artykuł|wpis|dokument|seria|geschichte|erzählung|erzahlung|artikel|serie)", RegexOptions.IgnoreCase); private static readonly Regex _PrevLink = new Regex("(prev|earl|[^b]old|new|wstecz|poprzednia|<|�)", RegexOptions.IgnoreCase); private static readonly Regex _PageRegex = new Regex("pag(e|ing|inat)|([^a-z]|^)pag([^a-z]|$)", RegexOptions.IgnoreCase); - private static readonly Regex _LikelyParagraphDivRegex = new Regex("text|para|parbase|paragraph", RegexOptions.IgnoreCase); + private static readonly Regex _LikelyParagraphDivRegex = new Regex("text|para|parbase|paragraph|figure", RegexOptions.IgnoreCase); + private static readonly Regex _LikelyImageContainerRegex = new Regex("image|figure|photo|media", RegexOptions.IgnoreCase); #endregion @@ -751,14 +752,9 @@ namespace ReadSharp.Ports.NReadability rootElement.GetElementsByTagName("script") .ForEach(scriptElement => - { - string scriptSrc = scriptElement.GetAttributeValue("src", null); - - if (string.IsNullOrEmpty(scriptSrc) || scriptSrc.LastIndexOf("readability") == -1) - { - elementsToRemove.Add(scriptElement); - } - }); + { + elementsToRemove.Add(scriptElement); + }); RemoveElements(elementsToRemove); @@ -1260,6 +1256,7 @@ namespace ReadSharp.Ports.NReadability /* Clean out junk from the article content. */ Clean(articleContentElement, "form"); Clean(articleContentElement, "object"); + Clean(articleContentElement, "meta"); if (articleContentElement.GetElementsByTagName("h1").Count() == 1) { @@ -1279,7 +1276,7 @@ namespace ReadSharp.Ports.NReadability /* Do these last as the previous stuff may have removed junk that will affect these. */ CleanConditionally(articleContentElement, "table"); CleanConditionally(articleContentElement, "ul"); - CleanConditionally(articleContentElement, "div"); + CleanConditionally(articleContentElement, "div", true); /* Remove extra paragraphs. */ IEnumerable paraElements = articleContentElement.GetElementsByTagName("p"); @@ -1460,7 +1457,7 @@ namespace ReadSharp.Ports.NReadability /// Cleans a of all elements with name if they look fishy. /// "Fishy" is an algorithm based on content length, classnames, link density, number of images and embeds, etc. /// - internal void CleanConditionally(XElement rootElement, string elementName) + internal void CleanConditionally(XElement rootElement, string elementName, bool checkImageContainer = false) { if (elementName == null) { @@ -1498,6 +1495,11 @@ namespace ReadSharp.Ports.NReadability int lisCount = element.GetElementsByTagName("li").Count(); int inputsCount = element.GetElementsByTagName("input").Count(); + if (checkImageContainer && imgsCount > 0 && _LikelyImageContainerRegex.IsMatch(element.GetClass())) + { + continue; + } + // while counting embeds we omit video-embeds int embedsCount = element.GetElementsByTagName("embed") diff --git a/ReadSharp.Tests/ReadTests.cs b/ReadSharp.Tests/ReadTests.cs index d03392e..8bbd176 100644 --- a/ReadSharp.Tests/ReadTests.cs +++ b/ReadSharp.Tests/ReadTests.cs @@ -1,4 +1,5 @@ using System; +using System.Linq; using System.Threading.Tasks; using Xunit; @@ -137,14 +138,17 @@ namespace ReadSharp.Tests [Fact] public async Task TestCriticalURIs() { - Article result = await reader.Read(new Uri("http://wpcentral.com.feedsportal.com/c/33999/f/616880/s/35a02b5e/sc/15/l/0L0Swpcentral0N0Cgameloft0Ediscusses0Etheir0Enew0Egame0Ebrothers0Earms0E30Esons0Ewar0Eceslive/story01.htm")); - Assert.NotEmpty(result.Content); + //Article result = await reader.Read(new Uri("http://wpcentral.com.feedsportal.com/c/33999/f/616880/s/35a02b5e/sc/15/l/0L0Swpcentral0N0Cgameloft0Ediscusses0Etheir0Enew0Egame0Ebrothers0Earms0E30Esons0Ewar0Eceslive/story01.htm")); + //Assert.NotEmpty(result.Content); - result = await reader.Read(new Uri("http://msdn.microsoft.com/en-us/library/windows/apps/hh464925.aspx")); - Assert.NotEmpty(result.Content); + //result = await reader.Read(new Uri("http://msdn.microsoft.com/en-us/library/windows/apps/hh464925.aspx")); + //Assert.NotEmpty(result.Content); - result = await reader.Read(new Uri("http://bit.ly/KAh7FJ")); - Assert.NotEmpty(result.Content); + //result = await reader.Read(new Uri("http://bit.ly/KAh7FJ")); + //Assert.NotEmpty(result.Content); + + Article result = await reader.Read(new Uri("http://www.nytimes.com/2014/01/31/world/europe/ukraine-unrest.html?hp&_r=0")); + Assert.True(result.Images != null && result.Images.Count > 0 && result.Images.SingleOrDefault(item => item.Uri.OriginalString == "http://static01.nyt.com/images/2014/01/31/world/31ukraine-cnd01/31ukraine-cnd01-articleLarge.jpg") != null); } [Fact]