diff --git a/CHANGELOG.md b/CHANGELOG.md index ba8c319..d98553a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +### 6.2.2 + +- Option (`PreferHTMLEncoding`) to either prefer HTML or HTTP encoding for generating content + ### 6.2.0 - Option to replace images with placeholders diff --git a/README.md b/README.md index 715a1ca..1352a6e 100644 --- a/README.md +++ b/README.md @@ -65,6 +65,7 @@ There are also `ReadOptions` available, which are passed on every request: - `bool` **HasNoHeadline**
Removes `

` title from the article - `bool` **UseDeepLinks**
If you check this option, deep-links (containing hashes, e.g. `href="#article"`) are not transformed into absolute URIs - `bool` **PrettyPrint**
Determines whether the HTML output will be formatted +- `bool` **PreferHTMLEncoding**
Determines whether to prefer the encoding found in the HTML or the one found in the HTTP Header (default: true) - `bool` **MultipageDownload**
Download all pages for articles with multiple pages (default: false) - `bool` **ReplaceImagesWithPlaceholders**
If true, replace all img-tags with placeholders diff --git a/ReadSharp.Tests/ReadTests.cs b/ReadSharp.Tests/ReadTests.cs index eb200d5..808f2e3 100644 --- a/ReadSharp.Tests/ReadTests.cs +++ b/ReadSharp.Tests/ReadTests.cs @@ -254,6 +254,8 @@ namespace ReadSharp.Tests [Fact] public async Task TestCriticalURIs2() { + ReadOptions options; + Article result = await reader.Read(new Uri("https://medium.com/best-thing-i-found-online-today/9e7455ca375b")); Assert.Contains("16. Be confident in how you ask", result.Content); @@ -269,6 +271,13 @@ namespace ReadSharp.Tests result = await reader.Read(new Uri("http://www.youtube.com/watch?v=GI2lHSPkW1c")); Assert.Contains("IT PAST MIDNIGHT A COUPLE HOURS AGO, IT'S FEELS COLDER", result.Content); + result = await reader.Read(new Uri("http://www.jn.pt/PaginaInicial/Politica/Interior.aspx?content_id=3996648&utm_source=feedburner&utm_medium=feed&utm_campaign=Feed%3A+JN-ULTIMAS+%28JN+-+Ultimas%29")); + Assert.DoesNotContain("Alberto João Jardim", result.Content); + + options = ReadOptions.CreateDefault(); + options.PreferHTMLEncoding = false; + result = await reader.Read(new Uri("http://www.jn.pt/PaginaInicial/Politica/Interior.aspx?content_id=3996648&utm_source=feedburner&utm_medium=feed&utm_campaign=Feed%3A+JN-ULTIMAS+%28JN+-+Ultimas%29"), options); + Assert.Contains("Alberto João Jardim", result.Content); } diff --git a/ReadSharp/Models/ReadOptions.cs b/ReadSharp/Models/ReadOptions.cs index 4c9221e..61dcd90 100644 --- a/ReadSharp/Models/ReadOptions.cs +++ b/ReadSharp/Models/ReadOptions.cs @@ -35,6 +35,14 @@ namespace ReadSharp /// public bool PrettyPrint { get; set; } + /// + /// Determines whether to prefer the encoding found in the HTML or the one found in the HTTP Header (default: true). + /// + /// + /// true if [prefer HTML encoding]; otherwise, false. + /// + public bool PreferHTMLEncoding { get; set; } + /// /// Download all pages for articles with multiple pages (default: false). /// @@ -60,6 +68,7 @@ namespace ReadSharp HasHeadline = false, UseDeepLinks = false, PrettyPrint = false, + PreferHTMLEncoding = true, MultipageDownload = false, ReplaceImagesWithPlaceholders = false }; diff --git a/ReadSharp/Properties/AssemblyInfo.cs b/ReadSharp/Properties/AssemblyInfo.cs index 1d665bf..c00f2b6 100644 --- a/ReadSharp/Properties/AssemblyInfo.cs +++ b/ReadSharp/Properties/AssemblyInfo.cs @@ -22,5 +22,5 @@ // You can specify all the values or you can default the Build and Revision Numbers // by using the '*' as shown below: // [assembly: AssemblyVersion("1.0.*")] -[assembly: AssemblyVersion("6.2.1")] -[assembly: AssemblyFileVersion("6.2.1")] \ No newline at end of file +[assembly: AssemblyVersion("6.2.2")] +[assembly: AssemblyFileVersion("6.2.2")] \ No newline at end of file diff --git a/ReadSharp/Reader.cs b/ReadSharp/Reader.cs index f545015..fa22cfb 100644 --- a/ReadSharp/Reader.cs +++ b/ReadSharp/Reader.cs @@ -349,15 +349,15 @@ namespace ReadSharp transcodingResult = ExtractReadableInformation(uri, responseStream, options, encoding); // get encoding found in HTML - encoding = _encoder.GetEncodingFromString(transcodingResult.Charset); + Encoding encodingFromHTML = _encoder.GetEncodingFromString(transcodingResult.Charset); // extract again if encoding didn't match or failed to retrieve - if (encoding != null && ( - String.IsNullOrEmpty(charset) + if ((encoding != null && String.IsNullOrEmpty(charset)) || - !String.Equals(charset, transcodingResult.Charset, StringComparison.OrdinalIgnoreCase))) + (options.PreferHTMLEncoding && !String.Equals(charset, transcodingResult.Charset, StringComparison.OrdinalIgnoreCase))) { - transcodingResult = ExtractReadableInformation(uri, responseStream, options, encoding); + transcodingResult = ExtractReadableInformation(uri, responseStream, options, encodingFromHTML); + encoding = encodingFromHTML; } } catch (Exception exc)