diff --git a/PortablePorts/NReadability/MetaExtractor.cs b/PortablePorts/NReadability/MetaExtractor.cs
index 3632226..4984c2d 100644
--- a/PortablePorts/NReadability/MetaExtractor.cs
+++ b/PortablePorts/NReadability/MetaExtractor.cs
@@ -103,22 +103,64 @@ namespace ReadSharp.Ports.NReadability
}
+ ///
+ /// Gets the charset.
+ ///
+ ///
+ public string GetCharset()
+ {
+ // find:
+ string result = SearchCandidates(new Dictionary()
+ {
+ { "charset", "charset" }
+ }, true);
+
+ if (String.IsNullOrEmpty(result))
+ {
+ // find:
+ result = SearchCandidates(new Dictionary()
+ {
+ { "http-equiv|Content-Type", "content" }
+ });
+
+ int charsetStart = result.IndexOf("charset=");
+ if (charsetStart > 0)
+ {
+ charsetStart += 8;
+ result = result.Substring(charsetStart, result.Length - charsetStart);
+ }
+ }
+
+ return String.IsNullOrEmpty(result) ? null : result.ToUpper();
+ }
+
+
///
/// Searches the candidates.
///
/// The candidates.
///
- private string SearchCandidates(Dictionary candidates)
+ private string SearchCandidates(Dictionary candidates, bool simple = false)
{
string result = null;
foreach (var candidate in candidates)
{
+ XElement element;
string[] type = candidate.Key.Split('|');
- XElement element = Tags
- .Where(item => String.Equals(item.GetAttributeValue(type[0], null), type[1], StringComparison.OrdinalIgnoreCase))
- .FirstOrDefault();
+ if (simple)
+ {
+ element = Tags
+ .Where(item => item.GetAttributeValue(type[0], null) != null)
+ .FirstOrDefault();
+ }
+ else
+ {
+ element = Tags
+ .Where(item => String.Equals(item.GetAttributeValue(type[0], null), type[1], StringComparison.OrdinalIgnoreCase))
+ .FirstOrDefault();
+ }
if (element != null)
{
diff --git a/PortablePorts/NReadability/NReadabilityTranscoder.cs b/PortablePorts/NReadability/NReadabilityTranscoder.cs
index a04a70d..1180aee 100644
--- a/PortablePorts/NReadability/NReadabilityTranscoder.cs
+++ b/PortablePorts/NReadability/NReadabilityTranscoder.cs
@@ -252,12 +252,14 @@ namespace ReadSharp.Ports.NReadability
MetaExtractor metaExtractor = new MetaExtractor(transcodedXmlDocument);
+ string charset = null;
string description = null;
Uri image = null;
Uri favicon = null;
if (metaExtractor.HasValue)
{
+ charset = metaExtractor.GetCharset();
description = metaExtractor.GetMetaDescription();
string imageString = metaExtractor.GetMetaImage();
string faviconString = metaExtractor.GetMetaFavicon();
@@ -283,6 +285,7 @@ namespace ReadSharp.Ports.NReadability
ExtractedFavicon = favicon,
ExtractedImage = image,
NextPageUrl = nextPageUrl,
+ Charset = charset,
Images = images
};
}
diff --git a/PortablePorts/NReadability/TranscodingResult.cs b/PortablePorts/NReadability/TranscodingResult.cs
index a6bfff6..dab6206 100644
--- a/PortablePorts/NReadability/TranscodingResult.cs
+++ b/PortablePorts/NReadability/TranscodingResult.cs
@@ -28,6 +28,8 @@ namespace ReadSharp.Ports.NReadability
public Uri ExtractedFavicon { get; set; }
+ public string Charset { get; set; }
+
public XDocument RawDocument { get; set; }
public IEnumerable Images { get; set; }
diff --git a/ReadSharp.Tests/ReadTests.cs b/ReadSharp.Tests/ReadTests.cs
index cfe9f7d..8273480 100644
--- a/ReadSharp.Tests/ReadTests.cs
+++ b/ReadSharp.Tests/ReadTests.cs
@@ -73,5 +73,27 @@ namespace ReadSharp.Tests
Assert.True(result.Content.Substring(0, 4) == "
+
diff --git a/ReadSharp/Reader.cs b/ReadSharp/Reader.cs
index 4ca23e8..bd31963 100644
--- a/ReadSharp/Reader.cs
+++ b/ReadSharp/Reader.cs
@@ -1,10 +1,12 @@
using ReadSharp.Ports.NReadability;
using System;
using System.Collections.Generic;
+using System.IO;
using System.Linq;
using System.Net;
using System.Net.Http;
using System.Reflection;
+using System.Text;
using System.Threading.Tasks;
using System.Xml.Linq;
@@ -25,6 +27,12 @@ namespace ReadSharp
///
protected readonly HttpClient _httpClient;
+ ///
+ /// The NReadability transcoder
+ ///
+ protected NReadabilityTranscoder _transcoder;
+
+
///
/// Initializes a new instance of the class.
@@ -34,6 +42,16 @@ namespace ReadSharp
/// Request timeout (in seconds).
public Reader(string userAgent = null, HttpMessageHandler handler = null, int? timeout = null)
{
+ // initialize transcoder
+ _transcoder = new NReadabilityTranscoder(
+ dontStripUnlikelys: false,
+ dontNormalizeSpacesInTextContent: true,
+ dontWeightClasses: false,
+ readingStyle: ReadingStyle.Ebook,
+ readingMargin: ReadingMargin.Narrow,
+ readingSize: ReadingSize.Medium
+ );
+
// override user agent
if (!string.IsNullOrEmpty(userAgent))
{
@@ -58,6 +76,8 @@ namespace ReadSharp
// add accepted encodings
_httpClient.DefaultRequestHeaders.TryAddWithoutValidation("Accept-Encoding", "gzip,deflate");
+ _httpClient.DefaultRequestHeaders.TryAddWithoutValidation("Accept-Charset", "UTF-8");
+
// add user agent
string version = Assembly.GetExecutingAssembly().FullName.Split(',')[1].Split('=')[1];
_httpClient.DefaultRequestHeaders.TryAddWithoutValidation("User-Agent", String.Format(_userAgent, "; ReadSharp/" + version));
@@ -77,37 +97,23 @@ namespace ReadSharp
///
public async Task Read(Uri uri, bool bodyOnly = true, bool noHeadline = false)
{
- // initialize transcoder
- NReadabilityTranscoder transcoder = new NReadabilityTranscoder(
- dontStripUnlikelys: false,
- dontNormalizeSpacesInTextContent: true,
- dontWeightClasses: false,
- readingStyle: ReadingStyle.Ebook,
- readingMargin: ReadingMargin.Narrow,
- readingSize: ReadingSize.Medium
- );
-
// get HTML string from URI
- string htmlResponse = await Request(uri);
+ Response response = await Request(uri);
- // set properties for processing
- TranscodingInput transcodingInput = new TranscodingInput(htmlResponse)
+ // readability
+ TranscodingResult transcodingResult = ExtractReadableInformation(uri, response.Stream, bodyOnly, noHeadline);
+
+ Encoding encoding = GetEncodingFromString(transcodingResult.Charset);
+
+
+ // extract again if encoding didn't match or failed to retrieve
+ if (encoding != null && (
+ String.IsNullOrEmpty(response.Charset)
+ ||
+ !String.Equals(response.Charset, transcodingResult.Charset, StringComparison.OrdinalIgnoreCase)))
{
- Url = uri.ToString(),
- DomSerializationParams = new DomSerializationParams()
- {
- BodyOnly = bodyOnly,
- NoHeadline = noHeadline,
- PrettyPrint = true,
- DontIncludeContentTypeMetaElement = true,
- DontIncludeMobileSpecificMetaElements = true,
- DontIncludeDocTypeMetaElement = false,
- DontIncludeGeneratorMetaElement = true
- }
- };
-
- // process/transcode HTML
- TranscodingResult transcodingResult = transcoder.Transcode(transcodingInput);
+ transcodingResult = ExtractReadableInformation(uri, response.Stream, bodyOnly, noHeadline, encoding);
+ }
// get images from article
List images = transcodingResult.Images.Select(image =>
@@ -138,12 +144,55 @@ namespace ReadSharp
+ ///
+ /// Extracts the readable information.
+ ///
+ /// The URI.
+ /// The text stream.
+ /// if set to true [body only].
+ /// if set to true [no headline].
+ ///
+ protected TranscodingResult ExtractReadableInformation(
+ Uri uri,
+ Stream textStream,
+ bool bodyOnly = true,
+ bool noHeadline = false,
+ Encoding encoding = null)
+ {
+
+ // response stream to text
+ textStream.Position = 0;
+ StreamReader streamReader = new StreamReader(textStream, encoding ?? Encoding.UTF8);
+ string text = streamReader.ReadToEnd();
+
+ // set properties for processing
+ TranscodingInput transcodingInput = new TranscodingInput(text)
+ {
+ Url = uri.ToString(),
+ DomSerializationParams = new DomSerializationParams()
+ {
+ BodyOnly = bodyOnly,
+ NoHeadline = noHeadline,
+ PrettyPrint = true,
+ DontIncludeContentTypeMetaElement = true,
+ DontIncludeMobileSpecificMetaElements = true,
+ DontIncludeDocTypeMetaElement = false,
+ DontIncludeGeneratorMetaElement = true
+ }
+ };
+
+ // process/transcode HTML
+ return _transcoder.Transcode(transcodingInput);
+ }
+
+
+
///
/// Fetches a resource
///
/// The URI.
///
- protected async Task Request(Uri uri)
+ private async Task Request(Uri uri)
{
HttpRequestMessage request = new HttpRequestMessage(HttpMethod.Get, uri);
HttpResponseMessage response = null;
@@ -167,7 +216,40 @@ namespace ReadSharp
}
// read response
- return await response.Content.ReadAsStringAsync();
+ Stream responseStream = await response.Content.ReadAsStreamAsync();
+
+ return new Response()
+ {
+ Stream = responseStream,
+ Charset = response.Content.Headers.ContentType.CharSet
+ };
+ }
+
+
+ ///
+ /// Gets the encoding from string.
+ ///
+ /// The encoding.
+ ///
+ private Encoding GetEncodingFromString(string encoding)
+ {
+ Encoding correctEncoding;
+
+ if (String.IsNullOrEmpty(encoding))
+ {
+ return null;
+ }
+
+ try
+ {
+ correctEncoding = Encoding.GetEncoding(encoding);
+ }
+ catch
+ {
+ return null;
+ }
+
+ return correctEncoding;
}
}
}
\ No newline at end of file