From febedfde06f4749c6a1728258877dde073221f55 Mon Sep 17 00:00:00 2001 From: Tobias Klika Date: Mon, 17 Feb 2014 14:34:01 +0100 Subject: [PATCH] multi-page download of articles --- ReadSharp.Tests/ReadTests.cs | 29 ++++- ReadSharp/Models/Article.cs | 8 ++ ReadSharp/Models/HttpOptions.cs | 15 ++- ReadSharp/Models/ReadOptions.cs | 19 +++- ReadSharp/Models/Response.cs | 10 +- ReadSharp/Reader.cs | 186 ++++++++++++++++++++------------ 6 files changed, 185 insertions(+), 82 deletions(-) diff --git a/ReadSharp.Tests/ReadTests.cs b/ReadSharp.Tests/ReadTests.cs index 6980a61..52e2bb2 100644 --- a/ReadSharp.Tests/ReadTests.cs +++ b/ReadSharp.Tests/ReadTests.cs @@ -169,6 +169,26 @@ namespace ReadSharp.Tests Assert.Null(result.NextPage); } + [Fact] + public async Task AreMultiPagesDownloadedAndMergedCorrectly() + { + ReadOptions options = new ReadOptions() { MultipageDownload = true }; + + Article result = await reader.Read(new Uri("http://www.zeit.de/gesellschaft/2014-02/alice-schwarzer-steuerhinterziehung-doppelmoral"), options); + Assert.Equal(result.PageCount, 2); + Assert.True(result.WordCount > 800); + + result = await reader.Read(new Uri("http://www.zeit.de/gesellschaft/2014-02/alice-schwarzer-steuerhinterziehung-doppelmoral")); + Assert.True(result.PageCount == 1 && result.WordCount < 500); + + result = await reader.Read(new Uri("http://arstechnica.com/apple/2014/01/two-steps-forward-a-review-of-the-2013-mac-pro/"), options); + Assert.Equal(result.PageCount, 7); + Assert.True(result.WordCount > 13000 && result.Images.Count > 10); + + result = await reader.Read(new Uri("http://www.sueddeutsche.de/wirtschaft/netzbetreiber-und-die-energiewende-im-kampf-gegen-blackouts-und-buergerproteste-1.1880754"), options); + Assert.Equal(result.PageCount, 2); + } + [Fact] public async Task TestCriticalURIs() { @@ -214,10 +234,13 @@ namespace ReadSharp.Tests [Fact] public async Task DebugArticle() { - string uri = "http://www.dgtle.com/article-5682-1.html"; + string uri = "http://www.zeit.de/gesellschaft/2014-02/alice-schwarzer-steuerhinterziehung-doppelmoral"; - Article result = await reader.Read(new Uri(uri)); - Assert.Contains("http://img.dgtle.com/forum/201402/13/162237x8oumb8i0i0y0087.jpeg!680px", result.Content); + Article result = await reader.Read(new Uri(uri), new ReadOptions() + { + MultipageDownload = true + }); + Assert.Equal(result.PageCount, 2); } } } \ No newline at end of file diff --git a/ReadSharp/Models/Article.cs b/ReadSharp/Models/Article.cs index 4662ab7..7597fe2 100644 --- a/ReadSharp/Models/Article.cs +++ b/ReadSharp/Models/Article.cs @@ -57,6 +57,14 @@ namespace ReadSharp /// public int WordCount { get; set; } + /// + /// Gets or sets the page count. + /// + /// + /// The page count. + /// + public int PageCount { get; set; } + /// /// Gets or sets the front image. /// diff --git a/ReadSharp/Models/HttpOptions.cs b/ReadSharp/Models/HttpOptions.cs index 2374547..d45278c 100644 --- a/ReadSharp/Models/HttpOptions.cs +++ b/ReadSharp/Models/HttpOptions.cs @@ -29,15 +29,23 @@ namespace ReadSharp public bool UseMobileUserAgent { get; set; } /// - /// Used UserAgent for HTTP request + /// Used UserAgent for HTTP request. /// public string UserAgent { get; set; } /// - /// Used mobile UserAgent for HTTP request + /// Used mobile UserAgent for HTTP request. /// public string UserAgentMobile { get; set; } + /// + /// Gets or sets the download limit for articles with multiple pages (default: 10). + /// + /// + /// The multipage limit. + /// + public int MultipageLimit { get; set; } + /// /// Creates the default HTTP options. /// @@ -50,7 +58,8 @@ namespace ReadSharp UserAgent = "Mozilla/5.0 (Windows NT 6.3; WOW64{0}) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1736.2 Safari/537.36 OPR/20.0.1380.1", UseMobileUserAgent = false, RequestTimeout = null, - CustomHttpHandler = null + CustomHttpHandler = null, + MultipageLimit = 10 }; } } diff --git a/ReadSharp/Models/ReadOptions.cs b/ReadSharp/Models/ReadOptions.cs index a67e41a..89708ba 100644 --- a/ReadSharp/Models/ReadOptions.cs +++ b/ReadSharp/Models/ReadOptions.cs @@ -4,7 +4,7 @@ namespace ReadSharp public class ReadOptions { /// - /// Are header tags and Doctype returned + /// Are header tags and Doctype returned (default: false). /// /// /// true if [has only body]; otherwise, false. @@ -12,7 +12,7 @@ namespace ReadSharp public bool HasHeaderTags { get; set; } /// - /// Is no headline (h1) is included in generated HTML + /// Is no headline (h1) is included in generated HTML (default: false). /// /// /// true if [has no headline]; otherwise, false. @@ -20,7 +20,7 @@ namespace ReadSharp public bool HasNoHeadline { get; set; } /// - /// Are deep links with hashes not transformed to absolute URIs + /// Are deep links with hashes not transformed to absolute URIs (default: false). /// /// /// true if [use deep links]; otherwise, false. @@ -28,13 +28,21 @@ namespace ReadSharp public bool UseDeepLinks { get; set; } /// - /// Determines whether the output will be formatted. + /// Determines whether the output will be formatted (default: false). /// /// /// true if [pretty print]; otherwise, false. /// public bool PrettyPrint { get; set; } + /// + /// Download all pages for articles with multiple pages (default: false). + /// + /// + /// true if [multipage download]; otherwise, false. + /// + public bool MultipageDownload { get; set; } + /// /// Creates the default options. /// @@ -46,7 +54,8 @@ namespace ReadSharp HasHeaderTags = false, HasNoHeadline = false, UseDeepLinks = false, - PrettyPrint = false + PrettyPrint = false, + MultipageDownload = false }; } } diff --git a/ReadSharp/Models/Response.cs b/ReadSharp/Models/Response.cs index 327de4f..544766e 100644 --- a/ReadSharp/Models/Response.cs +++ b/ReadSharp/Models/Response.cs @@ -1,14 +1,14 @@ -using System.IO; -using System.Net.Http; +using ReadSharp.Ports.NReadability; +using System.Text; namespace ReadSharp { internal class Response { - public HttpResponseMessage RawResponse { get; set; } + public TranscodingResult TranscodingResult { get; set; } - public Stream Stream { get; set; } + public Encoding Encoding { get; set; } - public string Charset { get; set; } + public int PageCount { get; set; } } } diff --git a/ReadSharp/Reader.cs b/ReadSharp/Reader.cs index a8338c4..73e188c 100644 --- a/ReadSharp/Reader.cs +++ b/ReadSharp/Reader.cs @@ -112,8 +112,6 @@ namespace ReadSharp public async Task
Read(Uri uri, ReadOptions options = null, CancellationToken cancellationToken = default(CancellationToken)) { Response response; - TranscodingResult transcodingResult; - Encoding encoding; string uriString = uri.OriginalString; if (options == null) @@ -131,61 +129,12 @@ namespace ReadSharp } // make async request - try - { - // get HTML string from URI - response = await Request(uri, cancellationToken); - } - catch (HttpRequestException exc) - { - throw new ReadException(exc.Message); - } - - // handle deep links - if (options.UseDeepLinks) - { - _transcoder.AnchorHrefTranformer = ReverseDeepLinks; - } - else - { - _transcoder.AnchorHrefTranformer = null; - } - - // readability - try - { - // charset found in HTTP headers - encoding = _encoder.GetEncodingFromString(response.Charset); - - // transcode content - transcodingResult = ExtractReadableInformation(uri, response.Stream, options, encoding); - - // get encoding found in HTML - encoding = _encoder.GetEncodingFromString(transcodingResult.Charset); - - // extract again if encoding didn't match or failed to retrieve - if (encoding != null && ( - String.IsNullOrEmpty(response.Charset) - || - !String.Equals(response.Charset, transcodingResult.Charset, StringComparison.OrdinalIgnoreCase))) - { - transcodingResult = ExtractReadableInformation(uri, response.Stream, options, encoding); - } - } - catch (Exception exc) - { - throw new ReadException(exc.Message); - } - finally - { - response.RawResponse.Dispose(); - response.Stream.Dispose(); - } + response = await Request(uri, options, null, cancellationToken); // get images from article int id = 1; - List images = transcodingResult.Images.Select(image => + List images = response.TranscodingResult.Images.Select(image => { Uri imageUri; Uri.TryCreate(image.GetAttributeValue("src", null), UriKind.Absolute, out imageUri); @@ -205,7 +154,7 @@ namespace ReadSharp try { - plainContent = HtmlUtilities.ConvertToPlainText(transcodingResult.ExtractedContent); + plainContent = HtmlUtilities.ConvertToPlainText(response.TranscodingResult.ExtractedContent); wordCount = HtmlUtilities.CountWords(plainContent); } catch @@ -216,17 +165,18 @@ namespace ReadSharp // create article return new Article() { - Title = transcodingResult.ExtractedTitle, - Description = transcodingResult.ExtractedDescription, - Content = transcodingResult.ExtractedContent, - ContentExtracted = transcodingResult.ContentExtracted ? wordCount > 0 : false, + Title = response.TranscodingResult.ExtractedTitle, + Description = response.TranscodingResult.ExtractedDescription, + Content = response.TranscodingResult.ExtractedContent, + ContentExtracted = response.TranscodingResult.ContentExtracted ? wordCount > 0 : false, PlainContent = plainContent, WordCount = wordCount, - FrontImage = transcodingResult.ExtractedImage, + PageCount = response.PageCount, + FrontImage = response.TranscodingResult.ExtractedImage, Images = images, - Favicon = transcodingResult.ExtractedFavicon, - NextPage = transcodingResult.NextPageUrl != null ? new Uri(transcodingResult.NextPageUrl, UriKind.Absolute) : null, - Encoding = _encoder.GetEncodingFromString(response.Charset) ?? encoding ?? null + Favicon = response.TranscodingResult.ExtractedFavicon, + NextPage = response.TranscodingResult.NextPageUrl != null ? new Uri(response.TranscodingResult.NextPageUrl, UriKind.Absolute) : null, + Encoding = response.Encoding }; } @@ -310,13 +260,17 @@ namespace ReadSharp /// Fetches a resource /// /// The URI. + /// The options. + /// if set to true [is continued page]. /// The cancellation token. /// /// /// - private async Task Request(Uri uri, CancellationToken cancellationToken) + private async Task Request(Uri uri, ReadOptions options, Response previousResponse, CancellationToken cancellationToken) { HttpResponseMessage response = null; + TranscodingResult transcodingResult; + Encoding encoding; using (HttpRequestMessage request = new HttpRequestMessage(HttpMethod.Get, uri)) { @@ -342,11 +296,111 @@ namespace ReadSharp // read response Stream responseStream = await response.Content.ReadAsStreamAsync(); + string charset = response.Content.Headers.ContentType.CharSet; + + // handle deep links + if (options.UseDeepLinks) + { + _transcoder.AnchorHrefTranformer = ReverseDeepLinks; + } + else + { + _transcoder.AnchorHrefTranformer = null; + } + + // readability + try + { + // charset found in HTTP headers + encoding = _encoder.GetEncodingFromString(charset); + + // transcode content + transcodingResult = ExtractReadableInformation(uri, responseStream, options, encoding); + + // get encoding found in HTML + encoding = _encoder.GetEncodingFromString(transcodingResult.Charset); + + // extract again if encoding didn't match or failed to retrieve + if (encoding != null && ( + String.IsNullOrEmpty(charset) + || + !String.Equals(charset, transcodingResult.Charset, StringComparison.OrdinalIgnoreCase))) + { + transcodingResult = ExtractReadableInformation(uri, responseStream, options, encoding); + } + } + catch (Exception exc) + { + throw new ReadException(exc.Message); + } + finally + { + response.Dispose(); + responseStream.Dispose(); + } + + Response newResponse = new Response() + { + TranscodingResult = transcodingResult, + PageCount = 1, + Encoding = encoding + }; + + // multiple pages are available + if (options.MultipageDownload && transcodingResult.NextPageUrl != null && (previousResponse == null || (previousResponse != null && previousResponse.PageCount < 10))) + { + return await Request(new Uri(transcodingResult.NextPageUrl), new ReadOptions() + { + HasNoHeadline = true, + PrettyPrint = options.PrettyPrint, + UseDeepLinks = options.UseDeepLinks, + MultipageDownload = true + }, previousResponse != null ? MergeResponses(previousResponse, newResponse) : newResponse, cancellationToken); + } + + // this is not the first page + if (previousResponse != null) + { + return MergeResponses(previousResponse, newResponse); + } + + return newResponse; + } + + + private Response MergeResponses(Response original, Response append) + { + if (original == null) + { + return append; + } + if (append == null) + { + return original; + } + + TranscodingResult mergedResult = original.TranscodingResult; + + mergedResult.ExtractedContent += String.Format("
{1}", (original.PageCount + 1).ToString(), append.TranscodingResult.ExtractedContent); + + if (mergedResult.Images == null || mergedResult.Images.Count() == 0) + { + mergedResult.Images = append.TranscodingResult.Images; + } + else if (append.TranscodingResult.Images != null && append.TranscodingResult.Images.Count() > 0) + { + List images = mergedResult.Images.ToList(); + images.AddRange(append.TranscodingResult.Images); + mergedResult.Images = images; + } + + mergedResult.NextPageUrl = append.TranscodingResult.NextPageUrl; + return new Response() { - RawResponse = response, - Stream = responseStream, - Charset = response.Content.Headers.ContentType.CharSet + PageCount = original.PageCount + 1, + Encoding = original.Encoding, + TranscodingResult = mergedResult }; } }