swcs/ReadSharp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

introduce HttpOptions for constructor; fix theVerge parsing

Browse Source
This commit is contained in:
Tobias Klika
2014-01-19 12:35:47 +01:00
parent 06dfa68323
commit 71eb9e9d66
6 changed files with 122 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files
PortablePorts/NReadability/DomExtensions.cs
+21
View File
@@ -282,6 +282,27 @@ namespace ReadSharp.Ports.NReadability
.Where(e => tagName.Equals(e.Name.LocalName, StringComparison.OrdinalIgnoreCase));
}
public static IEnumerable<XElement> GetElementsByClass(this XContainer container, string className)
{
if (container == null)
{
throw new ArgumentNullException("container");
}
if (string.IsNullOrEmpty(className))
{
throw new ArgumentNullException("className");
}
if (className.StartsWith("."))
{
className = className.Remove(0, 1);
}
return container.Descendants()
.Where(e => e != null && e.GetAttributeValue("class", "").Contains(className)); //tagName.Equals(e.Name.LocalName, StringComparison.OrdinalIgnoreCase));
}
public static IEnumerable<XElement> GetChildrenByTagName(this XContainer container, string tagName)
{
if (container == null)
PortablePorts/NReadability/NReadabilityTranscoder.cs
+18 -6
View File
@@ -20,6 +20,7 @@
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Reflection;
@@ -102,7 +103,7 @@ namespace ReadSharp.Ports.NReadability
private static readonly Regex _UnlikelyCandidatesRegex = new Regex("combx|comment|community|disqus|extra|foot|header|menu|remark|rss|shoutbox|sidebar|side|sponsor|ad-break|agegate|pagination|pager|popup|tweet|twitter", RegexOptions.IgnoreCase);
private static readonly Regex _OkMaybeItsACandidateRegex = new Regex("and|article|body|column|main|shadow", RegexOptions.IgnoreCase);
private static readonly Regex _PositiveWeightRegex = new Regex("article|body|content|entry|hentry|main|page|pagination|post|text|blog|story", RegexOptions.IgnoreCase);
private static readonly Regex _PositiveWeightRegex = new Regex("article|body|content|entry|hentry|main|page|pagination|post|text|blog|story|paragraph|instapaper_body|entry-content|article-body|entry-body", RegexOptions.IgnoreCase);
private static readonly Regex _NegativeWeightRegex = new Regex("combx|comment|com-|contact|foot|footer|footnote|masthead|media|meta|outbrain|promo|related|scroll|shoutbox|sidebar|side|sponsor|shopping|tags|tool|widget", RegexOptions.IgnoreCase);
private static readonly Regex _NegativeLinkParentRegex = new Regex("(stories|articles|news|documents|posts|notes|series|historie|artykuly|artykuły|wpisy|dokumenty|serie|geschichten|erzählungen|erzahlungen)", RegexOptions.IgnoreCase);
private static readonly Regex _Extraneous = new Regex("print|archive|comment|discuss|e[-]?mail|share|reply|all|login|sign|single|also", RegexOptions.IgnoreCase);
@@ -123,7 +124,7 @@ namespace ReadSharp.Ports.NReadability
private static readonly Regex _NextStoryLink = new Regex("(story|article|news|document|post|note|series|historia|artykul|artykuł|wpis|dokument|seria|geschichte|erzählung|erzahlung|artikel|serie)", RegexOptions.IgnoreCase);
private static readonly Regex _PrevLink = new Regex("(prev|earl|[^b]old|new|wstecz|poprzednia|<|�)", RegexOptions.IgnoreCase);
private static readonly Regex _PageRegex = new Regex("pag(e|ing|inat)|([^a-z]|^)pag([^a-z]|$)", RegexOptions.IgnoreCase);
private static readonly Regex _LikelyParagraphDivRegex = new Regex("text|para|parbase", RegexOptions.IgnoreCase);
private static readonly Regex _LikelyParagraphDivRegex = new Regex("text|para|parbase|paragraph", RegexOptions.IgnoreCase);
#endregion
@@ -135,7 +136,7 @@ namespace ReadSharp.Ports.NReadability
private static readonly Dictionary<Regex, string> _articleContentElementHints =
new Dictionary<Regex, string>
{
{ new Regex("^https?://(www|mobile)\\.theverge.com", RegexOptions.IgnoreCase), "article" },
{ new Regex("^https?://(www|mobile)\\.theverge.com", RegexOptions.IgnoreCase), ".entry-body" },
};
#endregion
@@ -1141,6 +1142,8 @@ namespace ReadSharp.Ports.NReadability
topCandidateElement.Add(documentBody.Nodes());
}
Debug.WriteLine(topCandidateElement);
return topCandidateElement;
}
@@ -1791,9 +1794,18 @@ namespace ReadSharp.Ports.NReadability
throw new ArgumentException("Argument can't be null nor empty.", "articleContentElementHint");
}
return document
.GetElementsByTagName(articleContentElementHint)
.FirstOrDefault();
if (articleContentElementHint.StartsWith("."))
{
return document
.GetElementsByClass(articleContentElementHint)
.FirstOrDefault();
}
else
{
return document
.GetElementsByTagName(articleContentElementHint)
.FirstOrDefault();
}
}
private static string GetArticleContentElementHint(string url)
ReadSharp.Tests/ReadTests.cs
+10
View File
@@ -1,4 +1,5 @@
using System;
using System.Diagnostics;
using System.Threading.Tasks;
using Xunit;
@@ -136,5 +137,14 @@ namespace ReadSharp.Tests
//Article result = await reader.Read(new Uri("http://bit.ly/KAh7FJ"));
//Assert.NotEmpty(result.Content);
}
[Fact]
public async Task TestVergeReturnsFullArticle()
{
Article result = await reader.Read(new Uri("http://www.theverge.com/2013/11/18/5116360/nokia-lumia-1520-review"));
Debug.WriteLine(result.Content.Length);
Assert.Contains("Three years ago, Nokia shipped over 110 million smartphones worldwide. ", result.Content);
Assert.True(result.Content.Length > 6000);
}
}
}
ReadSharp/Models/HttpOptions.cs
+57
View File
@@ -0,0 +1,57 @@
using System.Net.Http;
namespace ReadSharp
{
public class HttpOptions
{
/// <summary>
/// Gets or sets the custom HTTP handler.
/// </summary>
/// <value>
/// The custom HTTP handler.
/// </value>
public HttpMessageHandler CustomHttpHandler { get; set; }
/// <summary>
/// Gets or sets the request timeout.
/// </summary>
/// <value>
/// The timeout after which the requests cancels.
/// </value>
public int? RequestTimeout { get; set; }
/// <summary>
/// Gets or sets a value indicating whether [use mobile user agent].
/// </summary>
/// <value>
/// <c>true</c> if [use mobile user agent]; otherwise, <c>false</c>.
/// </value>
public bool UseMobileUserAgent { get; set; }
/// <summary>
/// Used UserAgent for HTTP request
/// </summary>
public string UserAgent { get; set; }
/// <summary>
/// Used mobile UserAgent for HTTP request
/// </summary>
public string UserAgentMobile { get; set; }
/// <summary>
/// Creates the default HTTP options.
/// </summary>
/// <returns></returns>
public static HttpOptions CreateDefault()
{
return new HttpOptions()
{
UserAgentMobile = "Mozilla/5.0 (Windows NT 6.3; Trident/7.0; rv:11.0; ARM; Mobile; Touch{0}) like Gecko",
UserAgent = "Mozilla/5.0 (Windows NT 6.3; WOW64{0}) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1736.2 Safari/537.36 OPR/20.0.1380.1",
UseMobileUserAgent = false,
RequestTimeout = null,
CustomHttpHandler = null
};
}
}
}
ReadSharp/ReadSharp.csproj
+1
View File
@@ -63,6 +63,7 @@
<Compile Include="IReader.cs" />
<Compile Include="Models\Article.cs" />
<Compile Include="Models\ArticleImage.cs" />
<Compile Include="Models\HttpOptions.cs" />
<Compile Include="Models\ReadOptions.cs" />
<Compile Include="Reader.cs" />
<Compile Include="Properties\AssemblyInfo.cs" />
ReadSharp/Reader.cs
+15 -19
View File
@@ -19,11 +19,6 @@ namespace ReadSharp
/// </summary>
public class Reader : IReader
{
/// <summary>
/// Used UserAgent for HTTP request
/// </summary>
protected string _userAgent = "Mozilla/5.0 (Windows NT 6.3; Trident/7.0; rv:11.0; ARM; Mobile; Touch{0}) like Gecko";
/// <summary>
/// REST client used for HTML retrieval
/// </summary>
@@ -44,10 +39,8 @@ namespace ReadSharp
/// <summary>
/// Initializes a new instance of the <see cref="Reader" /> class.
/// </summary>
/// <param name="userAgent">Custom UserAgent string.</param>
/// <param name="handler">The HttpMessage handler.</param>
/// <param name="timeout">Request timeout (in seconds).</param>
public Reader(string userAgent = null, HttpMessageHandler handler = null, int? timeout = null)
/// <param name="options">The HTTP options.</param>
public Reader(HttpOptions options = null)
{
// initialize transcoder
_transcoder = new NReadabilityTranscoder(
@@ -59,25 +52,25 @@ namespace ReadSharp
readingSize: ReadingSize.Medium
);
// get default HTTP options if none available
if (options == null)
{
options = HttpOptions.CreateDefault();
}
// initialize custom encoder
_encoder = new Encodings.Encoder(true);
// override user agent
if (!string.IsNullOrEmpty(userAgent))
{
_userAgent = userAgent;
}
// initialize HTTP client
_httpClient = new HttpClient(handler ?? new HttpClientHandler()
_httpClient = new HttpClient(options.CustomHttpHandler ?? new HttpClientHandler()
{
AutomaticDecompression = DecompressionMethods.Deflate | DecompressionMethods.GZip,
AllowAutoRedirect = true
});
if (timeout.HasValue)
if (options.RequestTimeout.HasValue)
{
_httpClient.Timeout = TimeSpan.FromSeconds(timeout.Value);
_httpClient.Timeout = TimeSpan.FromSeconds(options.RequestTimeout.Value);
}
// add accept types
@@ -87,8 +80,11 @@ namespace ReadSharp
_httpClient.DefaultRequestHeaders.TryAddWithoutValidation("Accept-Encoding", "gzip,deflate");
// add user agent
string userAgent = options.UseMobileUserAgent ? options.UserAgentMobile : options.UserAgent;
string version = Assembly.GetExecutingAssembly().FullName.Split(',')[1].Split('=')[1];
_httpClient.DefaultRequestHeaders.TryAddWithoutValidation("User-Agent", String.Format(_userAgent, "; ReadSharp/" + version));
_httpClient.DefaultRequestHeaders.TryAddWithoutValidation("User-Agent", String.Format(userAgent, "; ReadSharp/" + version));
}