swcs/ReadSharp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

multi-page download of articles

Browse Source
This commit is contained in:
Tobias Klika
2014-02-17 14:34:01 +01:00
parent 526abc27f5
commit febedfde06
6 changed files with 185 additions and 82 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ReadSharp.Tests/ReadTests.cs
+26 -3
View File
@@ -169,6 +169,26 @@ namespace ReadSharp.Tests
Assert.Null(result.NextPage);
}
[Fact]
public async Task AreMultiPagesDownloadedAndMergedCorrectly()
{
ReadOptions options = new ReadOptions() { MultipageDownload = true };
Article result = await reader.Read(new Uri("http://www.zeit.de/gesellschaft/2014-02/alice-schwarzer-steuerhinterziehung-doppelmoral"), options);
Assert.Equal(result.PageCount, 2);
Assert.True(result.WordCount > 800);
result = await reader.Read(new Uri("http://www.zeit.de/gesellschaft/2014-02/alice-schwarzer-steuerhinterziehung-doppelmoral"));
Assert.True(result.PageCount == 1 && result.WordCount < 500);
result = await reader.Read(new Uri("http://arstechnica.com/apple/2014/01/two-steps-forward-a-review-of-the-2013-mac-pro/"), options);
Assert.Equal(result.PageCount, 7);
Assert.True(result.WordCount > 13000 && result.Images.Count > 10);
result = await reader.Read(new Uri("http://www.sueddeutsche.de/wirtschaft/netzbetreiber-und-die-energiewende-im-kampf-gegen-blackouts-und-buergerproteste-1.1880754"), options);
Assert.Equal(result.PageCount, 2);
}
[Fact]
public async Task TestCriticalURIs()
{
@@ -214,10 +234,13 @@ namespace ReadSharp.Tests
[Fact]
public async Task DebugArticle()
{
string uri = "http://www.dgtle.com/article-5682-1.html";
string uri = "http://www.zeit.de/gesellschaft/2014-02/alice-schwarzer-steuerhinterziehung-doppelmoral";
Article result = await reader.Read(new Uri(uri));
Assert.Contains("http://img.dgtle.com/forum/201402/13/162237x8oumb8i0i0y0087.jpeg!680px", result.Content);
Article result = await reader.Read(new Uri(uri), new ReadOptions()
{
MultipageDownload = true
});
Assert.Equal(result.PageCount, 2);
}
}
}
ReadSharp/Models/Article.cs
+8
View File
@@ -57,6 +57,14 @@ namespace ReadSharp
/// </value>
public int WordCount { get; set; }
/// <summary>
/// Gets or sets the page count.
/// </summary>
/// <value>
/// The page count.
/// </value>
public int PageCount { get; set; }
/// <summary>
/// Gets or sets the front image.
/// </summary>
ReadSharp/Models/HttpOptions.cs
+12 -3
View File
@@ -29,15 +29,23 @@ namespace ReadSharp
public bool UseMobileUserAgent { get; set; }
/// <summary>
/// Used UserAgent for HTTP request
/// Used UserAgent for HTTP request.
/// </summary>
public string UserAgent { get; set; }
/// <summary>
/// Used mobile UserAgent for HTTP request
/// Used mobile UserAgent for HTTP request.
/// </summary>
public string UserAgentMobile { get; set; }
/// <summary>
/// Gets or sets the download limit for articles with multiple pages (default: 10).
/// </summary>
/// <value>
/// The multipage limit.
/// </value>
public int MultipageLimit { get; set; }
/// <summary>
/// Creates the default HTTP options.
/// </summary>
@@ -50,7 +58,8 @@ namespace ReadSharp
UserAgent = "Mozilla/5.0 (Windows NT 6.3; WOW64{0}) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1736.2 Safari/537.36 OPR/20.0.1380.1",
UseMobileUserAgent = false,
RequestTimeout = null,
CustomHttpHandler = null
CustomHttpHandler = null,
MultipageLimit = 10
};
}
}
ReadSharp/Models/ReadOptions.cs
+14 -5
View File
@@ -4,7 +4,7 @@ namespace ReadSharp
public class ReadOptions
{
/// <summary>
/// Are header tags and Doctype returned
/// Are header tags and Doctype returned (default: false).
/// </summary>
/// <value>
/// <c>true</c> if [has only body]; otherwise, <c>false</c>.
@@ -12,7 +12,7 @@ namespace ReadSharp
public bool HasHeaderTags { get; set; }
/// <summary>
/// Is no headline (h1) is included in generated HTML
/// Is no headline (h1) is included in generated HTML (default: false).
/// </summary>
/// <value>
/// <c>true</c> if [has no headline]; otherwise, <c>false</c>.
@@ -20,7 +20,7 @@ namespace ReadSharp
public bool HasNoHeadline { get; set; }
/// <summary>
/// Are deep links with hashes not transformed to absolute URIs
/// Are deep links with hashes not transformed to absolute URIs (default: false).
/// </summary>
/// <value>
/// <c>true</c> if [use deep links]; otherwise, <c>false</c>.
@@ -28,13 +28,21 @@ namespace ReadSharp
public bool UseDeepLinks { get; set; }
/// <summary>
/// Determines whether the output will be formatted.
/// Determines whether the output will be formatted (default: false).
/// </summary>
/// <value>
/// <c>true</c> if [pretty print]; otherwise, <c>false</c>.
/// </value>
public bool PrettyPrint { get; set; }
/// <summary>
/// Download all pages for articles with multiple pages (default: false).
/// </summary>
/// <value>
/// <c>true</c> if [multipage download]; otherwise, <c>false</c>.
/// </value>
public bool MultipageDownload { get; set; }
/// <summary>
/// Creates the default options.
/// </summary>
@@ -46,7 +54,8 @@ namespace ReadSharp
HasHeaderTags = false,
HasNoHeadline = false,
UseDeepLinks = false,
PrettyPrint = false
PrettyPrint = false,
MultipageDownload = false
};
}
}
ReadSharp/Models/Response.cs
+5 -5
View File
@@ -1,14 +1,14 @@
using System.IO;
using System.Net.Http;
using ReadSharp.Ports.NReadability;
using System.Text;
namespace ReadSharp
{
internal class Response
{
public HttpResponseMessage RawResponse { get; set; }
public TranscodingResult TranscodingResult { get; set; }
public Stream Stream { get; set; }
public Encoding Encoding { get; set; }
public string Charset { get; set; }
public int PageCount { get; set; }
}
}
ReadSharp/Reader.cs
+120 -66
View File
@@ -112,8 +112,6 @@ namespace ReadSharp
public async Task<Article> Read(Uri uri, ReadOptions options = null, CancellationToken cancellationToken = default(CancellationToken))
{
Response response;
TranscodingResult transcodingResult;
Encoding encoding;
string uriString = uri.OriginalString;
if (options == null)
@@ -131,61 +129,12 @@ namespace ReadSharp
}
// make async request
try
{
// get HTML string from URI
response = await Request(uri, cancellationToken);
}
catch (HttpRequestException exc)
{
throw new ReadException(exc.Message);
}
// handle deep links
if (options.UseDeepLinks)
{
_transcoder.AnchorHrefTranformer = ReverseDeepLinks;
}
else
{
_transcoder.AnchorHrefTranformer = null;
}
// readability
try
{
// charset found in HTTP headers
encoding = _encoder.GetEncodingFromString(response.Charset);
// transcode content
transcodingResult = ExtractReadableInformation(uri, response.Stream, options, encoding);
// get encoding found in HTML
encoding = _encoder.GetEncodingFromString(transcodingResult.Charset);
// extract again if encoding didn't match or failed to retrieve
if (encoding != null && (
String.IsNullOrEmpty(response.Charset)
||
!String.Equals(response.Charset, transcodingResult.Charset, StringComparison.OrdinalIgnoreCase)))
{
transcodingResult = ExtractReadableInformation(uri, response.Stream, options, encoding);
}
}
catch (Exception exc)
{
throw new ReadException(exc.Message);
}
finally
{
response.RawResponse.Dispose();
response.Stream.Dispose();
}
response = await Request(uri, options, null, cancellationToken);
// get images from article
int id = 1;
List<ArticleImage> images = transcodingResult.Images.Select<XElement, ArticleImage>(image =>
List<ArticleImage> images = response.TranscodingResult.Images.Select<XElement, ArticleImage>(image =>
{
Uri imageUri;
Uri.TryCreate(image.GetAttributeValue("src", null), UriKind.Absolute, out imageUri);
@@ -205,7 +154,7 @@ namespace ReadSharp
try
{
plainContent = HtmlUtilities.ConvertToPlainText(transcodingResult.ExtractedContent);
plainContent = HtmlUtilities.ConvertToPlainText(response.TranscodingResult.ExtractedContent);
wordCount = HtmlUtilities.CountWords(plainContent);
}
catch
@@ -216,17 +165,18 @@ namespace ReadSharp
// create article
return new Article()
{
Title = transcodingResult.ExtractedTitle,
Description = transcodingResult.ExtractedDescription,
Content = transcodingResult.ExtractedContent,
ContentExtracted = transcodingResult.ContentExtracted ? wordCount > 0 : false,
Title = response.TranscodingResult.ExtractedTitle,
Description = response.TranscodingResult.ExtractedDescription,
Content = response.TranscodingResult.ExtractedContent,
ContentExtracted = response.TranscodingResult.ContentExtracted ? wordCount > 0 : false,
PlainContent = plainContent,
WordCount = wordCount,
FrontImage = transcodingResult.ExtractedImage,
PageCount = response.PageCount,
FrontImage = response.TranscodingResult.ExtractedImage,
Images = images,
Favicon = transcodingResult.ExtractedFavicon,
NextPage = transcodingResult.NextPageUrl != null ? new Uri(transcodingResult.NextPageUrl, UriKind.Absolute) : null,
Encoding = _encoder.GetEncodingFromString(response.Charset) ?? encoding ?? null
Favicon = response.TranscodingResult.ExtractedFavicon,
NextPage = response.TranscodingResult.NextPageUrl != null ? new Uri(response.TranscodingResult.NextPageUrl, UriKind.Absolute) : null,
Encoding = response.Encoding
};
}
@@ -310,13 +260,17 @@ namespace ReadSharp
/// Fetches a resource
/// </summary>
/// <param name="uri">The URI.</param>
/// <param name="options">The options.</param>
/// <param name="isContinuedPage">if set to <c>true</c> [is continued page].</param>
/// <param name="cancellationToken">The cancellation token.</param>
/// <returns></returns>
/// <exception cref="ReadException">
/// </exception>
private async Task<Response> Request(Uri uri, CancellationToken cancellationToken)
private async Task<Response> Request(Uri uri, ReadOptions options, Response previousResponse, CancellationToken cancellationToken)
{
HttpResponseMessage response = null;
TranscodingResult transcodingResult;
Encoding encoding;
using (HttpRequestMessage request = new HttpRequestMessage(HttpMethod.Get, uri))
{
@@ -342,11 +296,111 @@ namespace ReadSharp
// read response
Stream responseStream = await response.Content.ReadAsStreamAsync();
string charset = response.Content.Headers.ContentType.CharSet;
// handle deep links
if (options.UseDeepLinks)
{
_transcoder.AnchorHrefTranformer = ReverseDeepLinks;
}
else
{
_transcoder.AnchorHrefTranformer = null;
}
// readability
try
{
// charset found in HTTP headers
encoding = _encoder.GetEncodingFromString(charset);
// transcode content
transcodingResult = ExtractReadableInformation(uri, responseStream, options, encoding);
// get encoding found in HTML
encoding = _encoder.GetEncodingFromString(transcodingResult.Charset);
// extract again if encoding didn't match or failed to retrieve
if (encoding != null && (
String.IsNullOrEmpty(charset)
||
!String.Equals(charset, transcodingResult.Charset, StringComparison.OrdinalIgnoreCase)))
{
transcodingResult = ExtractReadableInformation(uri, responseStream, options, encoding);
}
}
catch (Exception exc)
{
throw new ReadException(exc.Message);
}
finally
{
response.Dispose();
responseStream.Dispose();
}
Response newResponse = new Response()
{
TranscodingResult = transcodingResult,
PageCount = 1,
Encoding = encoding
};
// multiple pages are available
if (options.MultipageDownload && transcodingResult.NextPageUrl != null && (previousResponse == null || (previousResponse != null && previousResponse.PageCount < 10)))
{
return await Request(new Uri(transcodingResult.NextPageUrl), new ReadOptions()
{
HasNoHeadline = true,
PrettyPrint = options.PrettyPrint,
UseDeepLinks = options.UseDeepLinks,
MultipageDownload = true
}, previousResponse != null ? MergeResponses(previousResponse, newResponse) : newResponse, cancellationToken);
}
// this is not the first page
if (previousResponse != null)
{
return MergeResponses(previousResponse, newResponse);
}
return newResponse;
}
private Response MergeResponses(Response original, Response append)
{
if (original == null)
{
return append;
}
if (append == null)
{
return original;
}
TranscodingResult mergedResult = original.TranscodingResult;
mergedResult.ExtractedContent += String.Format("<div class=\"readability-nextpage\" data-page=\"{0}\"></div>{1}", (original.PageCount + 1).ToString(), append.TranscodingResult.ExtractedContent);
if (mergedResult.Images == null || mergedResult.Images.Count() == 0)
{
mergedResult.Images = append.TranscodingResult.Images;
}
else if (append.TranscodingResult.Images != null && append.TranscodingResult.Images.Count() > 0)
{
List<XElement> images = mergedResult.Images.ToList();
images.AddRange(append.TranscodingResult.Images);
mergedResult.Images = images;
}
mergedResult.NextPageUrl = append.TranscodingResult.NextPageUrl;
return new Response()
{
RawResponse = response,
Stream = responseStream,
Charset = response.Content.Headers.ContentType.CharSet
PageCount = original.PageCount + 1,
Encoding = original.Encoding,
TranscodingResult = mergedResult
};
}
}