fixes #12

Option (`PreferHTMLEncoding`) to either prefer HTML or HTTP encoding for
generating content

CHANGELOG.md

@@ -1,3 +1,7 @@
+### 6.2.2
+
+- Option (`PreferHTMLEncoding`) to either prefer HTML or HTTP encoding for generating content
+
 ### 6.2.0
 
 - Option to replace images with placeholders

README.md

@@ -65,6 +65,7 @@ There are also `ReadOptions` available, which are passed on every request:
 - `bool` **HasNoHeadline**<br>Removes `<h1>` title from the article
 - `bool` **UseDeepLinks**<br>If you check this option, deep-links (containing hashes, e.g. `href="#article"`) are not transformed into absolute URIs
 - `bool` **PrettyPrint**<br>Determines whether the HTML output will be formatted
+- `bool` **PreferHTMLEncoding**<br> Determines whether to prefer the encoding found in the HTML or the one found in the HTTP Header (default: true)
 - `bool` **MultipageDownload**<br>Download all pages for articles with multiple pages (default: false)
 - `bool` **ReplaceImagesWithPlaceholders**<br>If true, replace all img-tags with placeholders
 

ReadSharp.Tests/ReadTests.cs

@@ -254,6 +254,8 @@ namespace ReadSharp.Tests
     [Fact]
     public async Task TestCriticalURIs2()
     {
+      ReadOptions options;
+
       Article result = await reader.Read(new Uri("https://medium.com/best-thing-i-found-online-today/9e7455ca375b"));
       Assert.Contains("16. Be confident in how you ask", result.Content);
 
@@ -269,6 +271,13 @@ namespace ReadSharp.Tests
       result = await reader.Read(new Uri("http://www.youtube.com/watch?v=GI2lHSPkW1c"));
       Assert.Contains("IT PAST MIDNIGHT A COUPLE HOURS AGO, IT'S FEELS COLDER", result.Content);
 
+      result = await reader.Read(new Uri("http://www.jn.pt/PaginaInicial/Politica/Interior.aspx?content_id=3996648&utm_source=feedburner&utm_medium=feed&utm_campaign=Feed%3A+JN-ULTIMAS+%28JN+-+Ultimas%29"));
+      Assert.DoesNotContain("Alberto João Jardim", result.Content);
+
+      options = ReadOptions.CreateDefault();
+      options.PreferHTMLEncoding = false;
+      result = await reader.Read(new Uri("http://www.jn.pt/PaginaInicial/Politica/Interior.aspx?content_id=3996648&utm_source=feedburner&utm_medium=feed&utm_campaign=Feed%3A+JN-ULTIMAS+%28JN+-+Ultimas%29"), options);
+      Assert.Contains("Alberto João Jardim", result.Content);
     }
 
 

ReadSharp/Models/ReadOptions.cs

@@ -35,6 +35,14 @@ namespace ReadSharp
     /// </value>
     public bool PrettyPrint { get; set; }
 
+    /// <summary>
+    /// Determines whether to prefer the encoding found in the HTML or the one found in the HTTP Header (default: true).
+    /// </summary>
+    /// <value>
+    ///   <c>true</c> if [prefer HTML encoding]; otherwise, <c>false</c>.
+    /// </value>
+    public bool PreferHTMLEncoding { get; set; }
+
     /// <summary>
     /// Download all pages for articles with multiple pages (default: false).
     /// </summary>
@@ -60,6 +68,7 @@ namespace ReadSharp
         HasHeadline = false,
         UseDeepLinks = false,
         PrettyPrint = false,
+        PreferHTMLEncoding = true,
         MultipageDownload = false,
         ReplaceImagesWithPlaceholders = false
       };

ReadSharp/Properties/AssemblyInfo.cs

@@ -22,5 +22,5 @@
 // You can specify all the values or you can default the Build and Revision Numbers
 // by using the '*' as shown below:
 // [assembly: AssemblyVersion("1.0.*")]
-[assembly: AssemblyVersion("6.2.1")]
+[assembly: AssemblyVersion("6.2.2")]
-[assembly: AssemblyFileVersion("6.2.1")]
+[assembly: AssemblyFileVersion("6.2.2")]

ReadSharp/Reader.cs

@@ -349,15 +349,15 @@ namespace ReadSharp
         transcodingResult = ExtractReadableInformation(uri, responseStream, options, encoding);
 
         // get encoding found in HTML
-        encoding = _encoder.GetEncodingFromString(transcodingResult.Charset);
+        Encoding encodingFromHTML = _encoder.GetEncodingFromString(transcodingResult.Charset);
 
         // extract again if encoding didn't match or failed to retrieve
-        if (encoding != null && (
+        if ((encoding != null && String.IsNullOrEmpty(charset))
-          String.IsNullOrEmpty(charset)
           ||
-          !String.Equals(charset, transcodingResult.Charset, StringComparison.OrdinalIgnoreCase)))
+          (options.PreferHTMLEncoding && !String.Equals(charset, transcodingResult.Charset, StringComparison.OrdinalIgnoreCase)))
         {
-          transcodingResult = ExtractReadableInformation(uri, responseStream, options, encoding);
+          transcodingResult = ExtractReadableInformation(uri, responseStream, options, encodingFromHTML);
+          encoding = encodingFromHTML;
         }
       }
       catch (Exception exc)