diff --git a/PortablePorts/NReadability/DomExtensions.cs b/PortablePorts/NReadability/DomExtensions.cs index 2efe09b..edf42cd 100644 --- a/PortablePorts/NReadability/DomExtensions.cs +++ b/PortablePorts/NReadability/DomExtensions.cs @@ -282,6 +282,27 @@ namespace ReadSharp.Ports.NReadability .Where(e => tagName.Equals(e.Name.LocalName, StringComparison.OrdinalIgnoreCase)); } + public static IEnumerable GetElementsByClass(this XContainer container, string className) + { + if (container == null) + { + throw new ArgumentNullException("container"); + } + + if (string.IsNullOrEmpty(className)) + { + throw new ArgumentNullException("className"); + } + + if (className.StartsWith(".")) + { + className = className.Remove(0, 1); + } + + return container.Descendants() + .Where(e => e != null && e.GetAttributeValue("class", "").Contains(className)); //tagName.Equals(e.Name.LocalName, StringComparison.OrdinalIgnoreCase)); + } + public static IEnumerable GetChildrenByTagName(this XContainer container, string tagName) { if (container == null) diff --git a/PortablePorts/NReadability/NReadabilityTranscoder.cs b/PortablePorts/NReadability/NReadabilityTranscoder.cs index e1e542d..32ae6a6 100644 --- a/PortablePorts/NReadability/NReadabilityTranscoder.cs +++ b/PortablePorts/NReadability/NReadabilityTranscoder.cs @@ -20,6 +20,7 @@ using System; using System.Collections.Generic; +using System.Diagnostics; using System.IO; using System.Linq; using System.Reflection; @@ -102,7 +103,7 @@ namespace ReadSharp.Ports.NReadability private static readonly Regex _UnlikelyCandidatesRegex = new Regex("combx|comment|community|disqus|extra|foot|header|menu|remark|rss|shoutbox|sidebar|side|sponsor|ad-break|agegate|pagination|pager|popup|tweet|twitter", RegexOptions.IgnoreCase); private static readonly Regex _OkMaybeItsACandidateRegex = new Regex("and|article|body|column|main|shadow", RegexOptions.IgnoreCase); - private static readonly Regex _PositiveWeightRegex = new Regex("article|body|content|entry|hentry|main|page|pagination|post|text|blog|story", RegexOptions.IgnoreCase); + private static readonly Regex _PositiveWeightRegex = new Regex("article|body|content|entry|hentry|main|page|pagination|post|text|blog|story|paragraph|instapaper_body|entry-content|article-body|entry-body", RegexOptions.IgnoreCase); private static readonly Regex _NegativeWeightRegex = new Regex("combx|comment|com-|contact|foot|footer|footnote|masthead|media|meta|outbrain|promo|related|scroll|shoutbox|sidebar|side|sponsor|shopping|tags|tool|widget", RegexOptions.IgnoreCase); private static readonly Regex _NegativeLinkParentRegex = new Regex("(stories|articles|news|documents|posts|notes|series|historie|artykuly|artykuły|wpisy|dokumenty|serie|geschichten|erzählungen|erzahlungen)", RegexOptions.IgnoreCase); private static readonly Regex _Extraneous = new Regex("print|archive|comment|discuss|e[-]?mail|share|reply|all|login|sign|single|also", RegexOptions.IgnoreCase); @@ -123,7 +124,7 @@ namespace ReadSharp.Ports.NReadability private static readonly Regex _NextStoryLink = new Regex("(story|article|news|document|post|note|series|historia|artykul|artykuł|wpis|dokument|seria|geschichte|erzählung|erzahlung|artikel|serie)", RegexOptions.IgnoreCase); private static readonly Regex _PrevLink = new Regex("(prev|earl|[^b]old|new|wstecz|poprzednia|<|�)", RegexOptions.IgnoreCase); private static readonly Regex _PageRegex = new Regex("pag(e|ing|inat)|([^a-z]|^)pag([^a-z]|$)", RegexOptions.IgnoreCase); - private static readonly Regex _LikelyParagraphDivRegex = new Regex("text|para|parbase", RegexOptions.IgnoreCase); + private static readonly Regex _LikelyParagraphDivRegex = new Regex("text|para|parbase|paragraph", RegexOptions.IgnoreCase); #endregion @@ -135,7 +136,7 @@ namespace ReadSharp.Ports.NReadability private static readonly Dictionary _articleContentElementHints = new Dictionary { - { new Regex("^https?://(www|mobile)\\.theverge.com", RegexOptions.IgnoreCase), "article" }, + { new Regex("^https?://(www|mobile)\\.theverge.com", RegexOptions.IgnoreCase), ".entry-body" }, }; #endregion @@ -1141,6 +1142,8 @@ namespace ReadSharp.Ports.NReadability topCandidateElement.Add(documentBody.Nodes()); } + Debug.WriteLine(topCandidateElement); + return topCandidateElement; } @@ -1791,9 +1794,18 @@ namespace ReadSharp.Ports.NReadability throw new ArgumentException("Argument can't be null nor empty.", "articleContentElementHint"); } - return document - .GetElementsByTagName(articleContentElementHint) - .FirstOrDefault(); + if (articleContentElementHint.StartsWith(".")) + { + return document + .GetElementsByClass(articleContentElementHint) + .FirstOrDefault(); + } + else + { + return document + .GetElementsByTagName(articleContentElementHint) + .FirstOrDefault(); + } } private static string GetArticleContentElementHint(string url) diff --git a/ReadSharp.Tests/ReadTests.cs b/ReadSharp.Tests/ReadTests.cs index be50803..82492f1 100644 --- a/ReadSharp.Tests/ReadTests.cs +++ b/ReadSharp.Tests/ReadTests.cs @@ -1,4 +1,5 @@ using System; +using System.Diagnostics; using System.Threading.Tasks; using Xunit; @@ -136,5 +137,14 @@ namespace ReadSharp.Tests //Article result = await reader.Read(new Uri("http://bit.ly/KAh7FJ")); //Assert.NotEmpty(result.Content); } + + [Fact] + public async Task TestVergeReturnsFullArticle() + { + Article result = await reader.Read(new Uri("http://www.theverge.com/2013/11/18/5116360/nokia-lumia-1520-review")); + Debug.WriteLine(result.Content.Length); + Assert.Contains("Three years ago, Nokia shipped over 110 million smartphones worldwide. ", result.Content); + Assert.True(result.Content.Length > 6000); + } } } \ No newline at end of file diff --git a/ReadSharp/Models/HttpOptions.cs b/ReadSharp/Models/HttpOptions.cs new file mode 100644 index 0000000..2374547 --- /dev/null +++ b/ReadSharp/Models/HttpOptions.cs @@ -0,0 +1,57 @@ +using System.Net.Http; + +namespace ReadSharp +{ + public class HttpOptions + { + /// + /// Gets or sets the custom HTTP handler. + /// + /// + /// The custom HTTP handler. + /// + public HttpMessageHandler CustomHttpHandler { get; set; } + + /// + /// Gets or sets the request timeout. + /// + /// + /// The timeout after which the requests cancels. + /// + public int? RequestTimeout { get; set; } + + /// + /// Gets or sets a value indicating whether [use mobile user agent]. + /// + /// + /// true if [use mobile user agent]; otherwise, false. + /// + public bool UseMobileUserAgent { get; set; } + + /// + /// Used UserAgent for HTTP request + /// + public string UserAgent { get; set; } + + /// + /// Used mobile UserAgent for HTTP request + /// + public string UserAgentMobile { get; set; } + + /// + /// Creates the default HTTP options. + /// + /// + public static HttpOptions CreateDefault() + { + return new HttpOptions() + { + UserAgentMobile = "Mozilla/5.0 (Windows NT 6.3; Trident/7.0; rv:11.0; ARM; Mobile; Touch{0}) like Gecko", + UserAgent = "Mozilla/5.0 (Windows NT 6.3; WOW64{0}) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1736.2 Safari/537.36 OPR/20.0.1380.1", + UseMobileUserAgent = false, + RequestTimeout = null, + CustomHttpHandler = null + }; + } + } +} diff --git a/ReadSharp/ReadSharp.csproj b/ReadSharp/ReadSharp.csproj index cf9b0f3..800fdee 100644 --- a/ReadSharp/ReadSharp.csproj +++ b/ReadSharp/ReadSharp.csproj @@ -63,6 +63,7 @@ + diff --git a/ReadSharp/Reader.cs b/ReadSharp/Reader.cs index 2be19e5..4968bf1 100644 --- a/ReadSharp/Reader.cs +++ b/ReadSharp/Reader.cs @@ -19,11 +19,6 @@ namespace ReadSharp /// public class Reader : IReader { - /// - /// Used UserAgent for HTTP request - /// - protected string _userAgent = "Mozilla/5.0 (Windows NT 6.3; Trident/7.0; rv:11.0; ARM; Mobile; Touch{0}) like Gecko"; - /// /// REST client used for HTML retrieval /// @@ -44,10 +39,8 @@ namespace ReadSharp /// /// Initializes a new instance of the class. /// - /// Custom UserAgent string. - /// The HttpMessage handler. - /// Request timeout (in seconds). - public Reader(string userAgent = null, HttpMessageHandler handler = null, int? timeout = null) + /// The HTTP options. + public Reader(HttpOptions options = null) { // initialize transcoder _transcoder = new NReadabilityTranscoder( @@ -59,25 +52,25 @@ namespace ReadSharp readingSize: ReadingSize.Medium ); + // get default HTTP options if none available + if (options == null) + { + options = HttpOptions.CreateDefault(); + } + // initialize custom encoder _encoder = new Encodings.Encoder(true); - // override user agent - if (!string.IsNullOrEmpty(userAgent)) - { - _userAgent = userAgent; - } - // initialize HTTP client - _httpClient = new HttpClient(handler ?? new HttpClientHandler() + _httpClient = new HttpClient(options.CustomHttpHandler ?? new HttpClientHandler() { AutomaticDecompression = DecompressionMethods.Deflate | DecompressionMethods.GZip, AllowAutoRedirect = true }); - if (timeout.HasValue) + if (options.RequestTimeout.HasValue) { - _httpClient.Timeout = TimeSpan.FromSeconds(timeout.Value); + _httpClient.Timeout = TimeSpan.FromSeconds(options.RequestTimeout.Value); } // add accept types @@ -87,8 +80,11 @@ namespace ReadSharp _httpClient.DefaultRequestHeaders.TryAddWithoutValidation("Accept-Encoding", "gzip,deflate"); // add user agent + string userAgent = options.UseMobileUserAgent ? options.UserAgentMobile : options.UserAgent; + string version = Assembly.GetExecutingAssembly().FullName.Split(',')[1].Split('=')[1]; - _httpClient.DefaultRequestHeaders.TryAddWithoutValidation("User-Agent", String.Format(_userAgent, "; ReadSharp/" + version)); + + _httpClient.DefaultRequestHeaders.TryAddWithoutValidation("User-Agent", String.Format(userAgent, "; ReadSharp/" + version)); }