diff --git a/ReadSharp.Tests/ReadTests.cs b/ReadSharp.Tests/ReadTests.cs
index 6980a61..52e2bb2 100644
--- a/ReadSharp.Tests/ReadTests.cs
+++ b/ReadSharp.Tests/ReadTests.cs
@@ -169,6 +169,26 @@ namespace ReadSharp.Tests
Assert.Null(result.NextPage);
}
+ [Fact]
+ public async Task AreMultiPagesDownloadedAndMergedCorrectly()
+ {
+ ReadOptions options = new ReadOptions() { MultipageDownload = true };
+
+ Article result = await reader.Read(new Uri("http://www.zeit.de/gesellschaft/2014-02/alice-schwarzer-steuerhinterziehung-doppelmoral"), options);
+ Assert.Equal(result.PageCount, 2);
+ Assert.True(result.WordCount > 800);
+
+ result = await reader.Read(new Uri("http://www.zeit.de/gesellschaft/2014-02/alice-schwarzer-steuerhinterziehung-doppelmoral"));
+ Assert.True(result.PageCount == 1 && result.WordCount < 500);
+
+ result = await reader.Read(new Uri("http://arstechnica.com/apple/2014/01/two-steps-forward-a-review-of-the-2013-mac-pro/"), options);
+ Assert.Equal(result.PageCount, 7);
+ Assert.True(result.WordCount > 13000 && result.Images.Count > 10);
+
+ result = await reader.Read(new Uri("http://www.sueddeutsche.de/wirtschaft/netzbetreiber-und-die-energiewende-im-kampf-gegen-blackouts-und-buergerproteste-1.1880754"), options);
+ Assert.Equal(result.PageCount, 2);
+ }
+
[Fact]
public async Task TestCriticalURIs()
{
@@ -214,10 +234,13 @@ namespace ReadSharp.Tests
[Fact]
public async Task DebugArticle()
{
- string uri = "http://www.dgtle.com/article-5682-1.html";
+ string uri = "http://www.zeit.de/gesellschaft/2014-02/alice-schwarzer-steuerhinterziehung-doppelmoral";
- Article result = await reader.Read(new Uri(uri));
- Assert.Contains("http://img.dgtle.com/forum/201402/13/162237x8oumb8i0i0y0087.jpeg!680px", result.Content);
+ Article result = await reader.Read(new Uri(uri), new ReadOptions()
+ {
+ MultipageDownload = true
+ });
+ Assert.Equal(result.PageCount, 2);
}
}
}
\ No newline at end of file
diff --git a/ReadSharp/Models/Article.cs b/ReadSharp/Models/Article.cs
index 4662ab7..7597fe2 100644
--- a/ReadSharp/Models/Article.cs
+++ b/ReadSharp/Models/Article.cs
@@ -57,6 +57,14 @@ namespace ReadSharp
///
public int WordCount { get; set; }
+ ///
+ /// Gets or sets the page count.
+ ///
+ ///
+ /// The page count.
+ ///
+ public int PageCount { get; set; }
+
///
/// Gets or sets the front image.
///
diff --git a/ReadSharp/Models/HttpOptions.cs b/ReadSharp/Models/HttpOptions.cs
index 2374547..d45278c 100644
--- a/ReadSharp/Models/HttpOptions.cs
+++ b/ReadSharp/Models/HttpOptions.cs
@@ -29,15 +29,23 @@ namespace ReadSharp
public bool UseMobileUserAgent { get; set; }
///
- /// Used UserAgent for HTTP request
+ /// Used UserAgent for HTTP request.
///
public string UserAgent { get; set; }
///
- /// Used mobile UserAgent for HTTP request
+ /// Used mobile UserAgent for HTTP request.
///
public string UserAgentMobile { get; set; }
+ ///
+ /// Gets or sets the download limit for articles with multiple pages (default: 10).
+ ///
+ ///
+ /// The multipage limit.
+ ///
+ public int MultipageLimit { get; set; }
+
///
/// Creates the default HTTP options.
///
@@ -50,7 +58,8 @@ namespace ReadSharp
UserAgent = "Mozilla/5.0 (Windows NT 6.3; WOW64{0}) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1736.2 Safari/537.36 OPR/20.0.1380.1",
UseMobileUserAgent = false,
RequestTimeout = null,
- CustomHttpHandler = null
+ CustomHttpHandler = null,
+ MultipageLimit = 10
};
}
}
diff --git a/ReadSharp/Models/ReadOptions.cs b/ReadSharp/Models/ReadOptions.cs
index a67e41a..89708ba 100644
--- a/ReadSharp/Models/ReadOptions.cs
+++ b/ReadSharp/Models/ReadOptions.cs
@@ -4,7 +4,7 @@ namespace ReadSharp
public class ReadOptions
{
///
- /// Are header tags and Doctype returned
+ /// Are header tags and Doctype returned (default: false).
///
///
/// true if [has only body]; otherwise, false.
@@ -12,7 +12,7 @@ namespace ReadSharp
public bool HasHeaderTags { get; set; }
///
- /// Is no headline (h1) is included in generated HTML
+ /// Is no headline (h1) is included in generated HTML (default: false).
///
///
/// true if [has no headline]; otherwise, false.
@@ -20,7 +20,7 @@ namespace ReadSharp
public bool HasNoHeadline { get; set; }
///
- /// Are deep links with hashes not transformed to absolute URIs
+ /// Are deep links with hashes not transformed to absolute URIs (default: false).
///
///
/// true if [use deep links]; otherwise, false.
@@ -28,13 +28,21 @@ namespace ReadSharp
public bool UseDeepLinks { get; set; }
///
- /// Determines whether the output will be formatted.
+ /// Determines whether the output will be formatted (default: false).
///
///
/// true if [pretty print]; otherwise, false.
///
public bool PrettyPrint { get; set; }
+ ///
+ /// Download all pages for articles with multiple pages (default: false).
+ ///
+ ///
+ /// true if [multipage download]; otherwise, false.
+ ///
+ public bool MultipageDownload { get; set; }
+
///
/// Creates the default options.
///
@@ -46,7 +54,8 @@ namespace ReadSharp
HasHeaderTags = false,
HasNoHeadline = false,
UseDeepLinks = false,
- PrettyPrint = false
+ PrettyPrint = false,
+ MultipageDownload = false
};
}
}
diff --git a/ReadSharp/Models/Response.cs b/ReadSharp/Models/Response.cs
index 327de4f..544766e 100644
--- a/ReadSharp/Models/Response.cs
+++ b/ReadSharp/Models/Response.cs
@@ -1,14 +1,14 @@
-using System.IO;
-using System.Net.Http;
+using ReadSharp.Ports.NReadability;
+using System.Text;
namespace ReadSharp
{
internal class Response
{
- public HttpResponseMessage RawResponse { get; set; }
+ public TranscodingResult TranscodingResult { get; set; }
- public Stream Stream { get; set; }
+ public Encoding Encoding { get; set; }
- public string Charset { get; set; }
+ public int PageCount { get; set; }
}
}
diff --git a/ReadSharp/Reader.cs b/ReadSharp/Reader.cs
index a8338c4..73e188c 100644
--- a/ReadSharp/Reader.cs
+++ b/ReadSharp/Reader.cs
@@ -112,8 +112,6 @@ namespace ReadSharp
public async Task Read(Uri uri, ReadOptions options = null, CancellationToken cancellationToken = default(CancellationToken))
{
Response response;
- TranscodingResult transcodingResult;
- Encoding encoding;
string uriString = uri.OriginalString;
if (options == null)
@@ -131,61 +129,12 @@ namespace ReadSharp
}
// make async request
- try
- {
- // get HTML string from URI
- response = await Request(uri, cancellationToken);
- }
- catch (HttpRequestException exc)
- {
- throw new ReadException(exc.Message);
- }
-
- // handle deep links
- if (options.UseDeepLinks)
- {
- _transcoder.AnchorHrefTranformer = ReverseDeepLinks;
- }
- else
- {
- _transcoder.AnchorHrefTranformer = null;
- }
-
- // readability
- try
- {
- // charset found in HTTP headers
- encoding = _encoder.GetEncodingFromString(response.Charset);
-
- // transcode content
- transcodingResult = ExtractReadableInformation(uri, response.Stream, options, encoding);
-
- // get encoding found in HTML
- encoding = _encoder.GetEncodingFromString(transcodingResult.Charset);
-
- // extract again if encoding didn't match or failed to retrieve
- if (encoding != null && (
- String.IsNullOrEmpty(response.Charset)
- ||
- !String.Equals(response.Charset, transcodingResult.Charset, StringComparison.OrdinalIgnoreCase)))
- {
- transcodingResult = ExtractReadableInformation(uri, response.Stream, options, encoding);
- }
- }
- catch (Exception exc)
- {
- throw new ReadException(exc.Message);
- }
- finally
- {
- response.RawResponse.Dispose();
- response.Stream.Dispose();
- }
+ response = await Request(uri, options, null, cancellationToken);
// get images from article
int id = 1;
- List images = transcodingResult.Images.Select(image =>
+ List images = response.TranscodingResult.Images.Select(image =>
{
Uri imageUri;
Uri.TryCreate(image.GetAttributeValue("src", null), UriKind.Absolute, out imageUri);
@@ -205,7 +154,7 @@ namespace ReadSharp
try
{
- plainContent = HtmlUtilities.ConvertToPlainText(transcodingResult.ExtractedContent);
+ plainContent = HtmlUtilities.ConvertToPlainText(response.TranscodingResult.ExtractedContent);
wordCount = HtmlUtilities.CountWords(plainContent);
}
catch
@@ -216,17 +165,18 @@ namespace ReadSharp
// create article
return new Article()
{
- Title = transcodingResult.ExtractedTitle,
- Description = transcodingResult.ExtractedDescription,
- Content = transcodingResult.ExtractedContent,
- ContentExtracted = transcodingResult.ContentExtracted ? wordCount > 0 : false,
+ Title = response.TranscodingResult.ExtractedTitle,
+ Description = response.TranscodingResult.ExtractedDescription,
+ Content = response.TranscodingResult.ExtractedContent,
+ ContentExtracted = response.TranscodingResult.ContentExtracted ? wordCount > 0 : false,
PlainContent = plainContent,
WordCount = wordCount,
- FrontImage = transcodingResult.ExtractedImage,
+ PageCount = response.PageCount,
+ FrontImage = response.TranscodingResult.ExtractedImage,
Images = images,
- Favicon = transcodingResult.ExtractedFavicon,
- NextPage = transcodingResult.NextPageUrl != null ? new Uri(transcodingResult.NextPageUrl, UriKind.Absolute) : null,
- Encoding = _encoder.GetEncodingFromString(response.Charset) ?? encoding ?? null
+ Favicon = response.TranscodingResult.ExtractedFavicon,
+ NextPage = response.TranscodingResult.NextPageUrl != null ? new Uri(response.TranscodingResult.NextPageUrl, UriKind.Absolute) : null,
+ Encoding = response.Encoding
};
}
@@ -310,13 +260,17 @@ namespace ReadSharp
/// Fetches a resource
///
/// The URI.
+ /// The options.
+ /// if set to true [is continued page].
/// The cancellation token.
///
///
///
- private async Task Request(Uri uri, CancellationToken cancellationToken)
+ private async Task Request(Uri uri, ReadOptions options, Response previousResponse, CancellationToken cancellationToken)
{
HttpResponseMessage response = null;
+ TranscodingResult transcodingResult;
+ Encoding encoding;
using (HttpRequestMessage request = new HttpRequestMessage(HttpMethod.Get, uri))
{
@@ -342,11 +296,111 @@ namespace ReadSharp
// read response
Stream responseStream = await response.Content.ReadAsStreamAsync();
+ string charset = response.Content.Headers.ContentType.CharSet;
+
+ // handle deep links
+ if (options.UseDeepLinks)
+ {
+ _transcoder.AnchorHrefTranformer = ReverseDeepLinks;
+ }
+ else
+ {
+ _transcoder.AnchorHrefTranformer = null;
+ }
+
+ // readability
+ try
+ {
+ // charset found in HTTP headers
+ encoding = _encoder.GetEncodingFromString(charset);
+
+ // transcode content
+ transcodingResult = ExtractReadableInformation(uri, responseStream, options, encoding);
+
+ // get encoding found in HTML
+ encoding = _encoder.GetEncodingFromString(transcodingResult.Charset);
+
+ // extract again if encoding didn't match or failed to retrieve
+ if (encoding != null && (
+ String.IsNullOrEmpty(charset)
+ ||
+ !String.Equals(charset, transcodingResult.Charset, StringComparison.OrdinalIgnoreCase)))
+ {
+ transcodingResult = ExtractReadableInformation(uri, responseStream, options, encoding);
+ }
+ }
+ catch (Exception exc)
+ {
+ throw new ReadException(exc.Message);
+ }
+ finally
+ {
+ response.Dispose();
+ responseStream.Dispose();
+ }
+
+ Response newResponse = new Response()
+ {
+ TranscodingResult = transcodingResult,
+ PageCount = 1,
+ Encoding = encoding
+ };
+
+ // multiple pages are available
+ if (options.MultipageDownload && transcodingResult.NextPageUrl != null && (previousResponse == null || (previousResponse != null && previousResponse.PageCount < 10)))
+ {
+ return await Request(new Uri(transcodingResult.NextPageUrl), new ReadOptions()
+ {
+ HasNoHeadline = true,
+ PrettyPrint = options.PrettyPrint,
+ UseDeepLinks = options.UseDeepLinks,
+ MultipageDownload = true
+ }, previousResponse != null ? MergeResponses(previousResponse, newResponse) : newResponse, cancellationToken);
+ }
+
+ // this is not the first page
+ if (previousResponse != null)
+ {
+ return MergeResponses(previousResponse, newResponse);
+ }
+
+ return newResponse;
+ }
+
+
+ private Response MergeResponses(Response original, Response append)
+ {
+ if (original == null)
+ {
+ return append;
+ }
+ if (append == null)
+ {
+ return original;
+ }
+
+ TranscodingResult mergedResult = original.TranscodingResult;
+
+ mergedResult.ExtractedContent += String.Format("{1}", (original.PageCount + 1).ToString(), append.TranscodingResult.ExtractedContent);
+
+ if (mergedResult.Images == null || mergedResult.Images.Count() == 0)
+ {
+ mergedResult.Images = append.TranscodingResult.Images;
+ }
+ else if (append.TranscodingResult.Images != null && append.TranscodingResult.Images.Count() > 0)
+ {
+ List images = mergedResult.Images.ToList();
+ images.AddRange(append.TranscodingResult.Images);
+ mergedResult.Images = images;
+ }
+
+ mergedResult.NextPageUrl = append.TranscodingResult.NextPageUrl;
+
return new Response()
{
- RawResponse = response,
- Stream = responseStream,
- Charset = response.Content.Headers.ContentType.CharSet
+ PageCount = original.PageCount + 1,
+ Encoding = original.Encoding,
+ TranscodingResult = mergedResult
};
}
}