diff --git a/PortablePorts/NReadability/NReadabilityTranscoder.cs b/PortablePorts/NReadability/NReadabilityTranscoder.cs index e0ce027..dd96079 100644 --- a/PortablePorts/NReadability/NReadabilityTranscoder.cs +++ b/PortablePorts/NReadability/NReadabilityTranscoder.cs @@ -142,7 +142,9 @@ namespace ReadSharp.Ports.NReadability { new Regex("^https?://(www|blog)\\.bufferapp.com", RegexOptions.IgnoreCase), ".post" }, { new Regex("^https?://(www.)?polygon.com", RegexOptions.IgnoreCase), ".body" }, { new Regex("^https?://(www.)?gizmodo.com", RegexOptions.IgnoreCase), ".post-container" }, - { new Regex("^https?://(www.)?it-scoop.com", RegexOptions.IgnoreCase), ".entry-content" } + { new Regex("^https?://(www.)?it-scoop.com", RegexOptions.IgnoreCase), ".entry-content" }, + { new Regex("^https?://(m\\.|www\\.)?youtube.com", RegexOptions.IgnoreCase), "#watch-description-content" }, + { new Regex("^https?://(www.)?vimeo.com", RegexOptions.IgnoreCase), ".description_wrapper" } }; #endregion @@ -545,10 +547,29 @@ namespace ReadSharp.Ports.NReadability double urlComparity = StringCompare(url, linkHref); /* If the next page differs from the original too much */ - if (StringCompare(url, linkHref) < 80) + if (urlComparity < 80) { linkObj.Score -= 200; } + else if (urlComparity > 95 && url.Length <= linkHref.Length) + { + try + { + string originalLinkHrefNumber = Regex.Match(url.Replace(articleBaseUrl, ""), @"\d+").Value; + string linkHrefNumber = Regex.Match(linkHref.Replace(articleBaseUrl, ""), @"\d+").Value; + + if (originalLinkHrefNumber == "") + { + originalLinkHrefNumber = "0"; + } + + if (originalLinkHrefNumber != null && linkHrefNumber != null && linkHrefNumber.Length < 3 && linkHref.EndsWith(linkHrefNumber) && Convert.ToByte(originalLinkHrefNumber) < Convert.ToByte(linkHrefNumber)) + { + linkObj.Score += 75; + } + } + catch { } + } /* If any ancestor node contains page or paging or paginat */ XElement parentNode = linkElement.Parent; diff --git a/ReadSharp.Tests/ReadTests.cs b/ReadSharp.Tests/ReadTests.cs index 52e2bb2..a38d6c5 100644 --- a/ReadSharp.Tests/ReadTests.cs +++ b/ReadSharp.Tests/ReadTests.cs @@ -22,7 +22,6 @@ namespace ReadSharp.Tests Article result = await reader.Read(new Uri("http://frontendplay.com/story/4/http-caching-demystified-part-2-implementation")); Assert.DoesNotContain("", result.Content); - Assert.Contains("

", result.Content); Assert.True(result.Content.Length > 15000); } @@ -32,7 +31,6 @@ namespace ReadSharp.Tests { Article result = await reader.Read(new Uri("http://frontendplay.com/story/4/http-caching-demystified-part-2-implementation"), new ReadOptions() { - HasNoHeadline = true, HasHeaderTags = true }); @@ -143,7 +141,10 @@ namespace ReadSharp.Tests [Fact] public async Task AreMultipageArticlesWorking() { - Article result = await reader.Read(new Uri("http://www.zeit.de/gesellschaft/2014-02/alice-schwarzer-steuerhinterziehung-doppelmoral")); + Article result = await reader.Read(new Uri("http://www.anandtech.com/show/7594/samsung-ssd-840-evo-msata-120gb-250gb-500gb-1tb-review")); + Assert.Equal(result.NextPage.ToString(), "http://www.anandtech.com/show/7594/samsung-ssd-840-evo-msata-120gb-250gb-500gb-1tb-review/2"); + + result = await reader.Read(new Uri("http://www.zeit.de/gesellschaft/2014-02/alice-schwarzer-steuerhinterziehung-doppelmoral")); Assert.Equal(result.NextPage.ToString(), "http://www.zeit.de/gesellschaft/2014-02/alice-schwarzer-steuerhinterziehung-doppelmoral/seite-2"); result = await reader.Read(new Uri("http://www.sueddeutsche.de/wirtschaft/netzbetreiber-und-die-energiewende-im-kampf-gegen-blackouts-und-buergerproteste-1.1880754")); @@ -156,10 +157,14 @@ namespace ReadSharp.Tests [Fact] public async Task AreSinglepageArticlesNotPopulatingNextPage() { - Article result = await reader.Read(new Uri("http://www.buzzfeed.com/mattlynley/the-16-most-interesting-things-to-come-out-of-bill-gates-qa")); + Article result = await reader.Read(new Uri("http://arstechnica.com/apple/2014/01/two-steps-forward-a-review-of-the-2013-mac-pro/7/")); Assert.Null(result.NextPage); - result = await reader.Read(new Uri("http://arstechnica.com/apple/2014/01/two-steps-forward-a-review-of-the-2013-mac-pro/7/")); + result = await reader.Read(new Uri("http://www.wpcentral.com/developers-leak-new-features-windows-phone-81-sdk"), new ReadOptions() { MultipageDownload = true }); + Assert.Null(result.NextPage); + Assert.Equal(result.PageCount, 1); + + result = await reader.Read(new Uri("http://www.buzzfeed.com/mattlynley/the-16-most-interesting-things-to-come-out-of-bill-gates-qa")); Assert.Null(result.NextPage); result = await reader.Read(new Uri("http://www.sueddeutsche.de/wirtschaft/netzbetreiber-und-die-energiewende-im-kampf-gegen-blackouts-und-buergerproteste-1.1880754-2")); @@ -167,6 +172,7 @@ namespace ReadSharp.Tests result = await reader.Read(new Uri("http://www.zeit.de/gesellschaft/2014-02/alice-schwarzer-steuerhinterziehung-doppelmoral/seite-2")); Assert.Null(result.NextPage); + } [Fact] @@ -174,7 +180,10 @@ namespace ReadSharp.Tests { ReadOptions options = new ReadOptions() { MultipageDownload = true }; - Article result = await reader.Read(new Uri("http://www.zeit.de/gesellschaft/2014-02/alice-schwarzer-steuerhinterziehung-doppelmoral"), options); + Article result = await reader.Read(new Uri("http://www.anandtech.com/show/7594/samsung-ssd-840-evo-msata-120gb-250gb-500gb-1tb-review"), options); + Assert.Equal(result.PageCount, 9); + + result = await reader.Read(new Uri("http://www.zeit.de/gesellschaft/2014-02/alice-schwarzer-steuerhinterziehung-doppelmoral"), options); Assert.Equal(result.PageCount, 2); Assert.True(result.WordCount > 800); @@ -228,6 +237,12 @@ namespace ReadSharp.Tests result = await reader.Read(new Uri("http://m.spiegel.de/spiegelgeschichte/a-946060.html")); Assert.DoesNotContain("Detecting browser settings", result.Content); + + result = await reader.Read(new Uri("https://vimeo.com/84391640")); + Assert.Contains("twitter.com/pokiapp", result.Content); + + result = await reader.Read(new Uri("http://www.youtube.com/watch?v=GI2lHSPkW1c")); + Assert.Contains("IT PAST MIDNIGHT A COUPLE HOURS AGO, IT'S FEELS COLDER", result.Content); } diff --git a/ReadSharp/HtmlUtilities.cs b/ReadSharp/HtmlUtilities.cs index 60911c6..579b746 100644 --- a/ReadSharp/HtmlUtilities.cs +++ b/ReadSharp/HtmlUtilities.cs @@ -94,6 +94,9 @@ namespace ReadSharp // treat paragraphs as crlf outText.Write("\r\n"); break; + case "br": + outText.Write("\r\n"); + break; } if (node.HasChildNodes) diff --git a/ReadSharp/Models/ReadOptions.cs b/ReadSharp/Models/ReadOptions.cs index 89708ba..1425d58 100644 --- a/ReadSharp/Models/ReadOptions.cs +++ b/ReadSharp/Models/ReadOptions.cs @@ -17,7 +17,7 @@ namespace ReadSharp /// /// true if [has no headline]; otherwise, false. /// - public bool HasNoHeadline { get; set; } + public bool HasHeadline { get; set; } /// /// Are deep links with hashes not transformed to absolute URIs (default: false). @@ -52,7 +52,7 @@ namespace ReadSharp return new ReadOptions() { HasHeaderTags = false, - HasNoHeadline = false, + HasHeadline = false, UseDeepLinks = false, PrettyPrint = false, MultipageDownload = false diff --git a/ReadSharp/Reader.cs b/ReadSharp/Reader.cs index d824bfe..f14e0a6 100644 --- a/ReadSharp/Reader.cs +++ b/ReadSharp/Reader.cs @@ -214,7 +214,7 @@ namespace ReadSharp DomSerializationParams = new DomSerializationParams() { BodyOnly = !options.HasHeaderTags, - NoHeadline = options.HasNoHeadline, + NoHeadline = !options.HasHeadline, PrettyPrint = options.PrettyPrint, DontIncludeContentTypeMetaElement = true, DontIncludeMobileSpecificMetaElements = true, @@ -352,6 +352,14 @@ namespace ReadSharp Encoding = encoding }; + // in same special cases their are multiple pages, which are only comments or do not contain new content. + // if this is the case we will break here and return the first page only. + if (previousResponse != null && previousResponse.TranscodingResult.ExtractedContent == transcodingResult.ExtractedContent) + { + previousResponse.TranscodingResult.NextPageUrl = null; + return previousResponse; + } + // multiple pages are available try { @@ -359,7 +367,6 @@ namespace ReadSharp { return await Request(new Uri(transcodingResult.NextPageUrl), new ReadOptions() { - HasNoHeadline = true, PrettyPrint = options.PrettyPrint, UseDeepLinks = options.UseDeepLinks, MultipageDownload = true