From cebb186db69c61f5092e40190e3d657535506d47 Mon Sep 17 00:00:00 2001 From: Tobias Klika Date: Thu, 19 Dec 2013 21:13:23 +0100 Subject: [PATCH] fix charset bugs --- PortablePorts/NReadability/MetaExtractor.cs | 50 +++++- .../NReadability/NReadabilityTranscoder.cs | 3 + .../NReadability/TranscodingResult.cs | 2 + ReadSharp.Tests/ReadTests.cs | 22 +++ ReadSharp/Models/Response.cs | 11 ++ ReadSharp/Properties/AssemblyInfo.cs | 4 +- ReadSharp/ReadSharp.csproj | 1 + ReadSharp/Reader.cs | 142 ++++++++++++++---- 8 files changed, 199 insertions(+), 36 deletions(-) create mode 100644 ReadSharp/Models/Response.cs diff --git a/PortablePorts/NReadability/MetaExtractor.cs b/PortablePorts/NReadability/MetaExtractor.cs index 3632226..4984c2d 100644 --- a/PortablePorts/NReadability/MetaExtractor.cs +++ b/PortablePorts/NReadability/MetaExtractor.cs @@ -103,22 +103,64 @@ namespace ReadSharp.Ports.NReadability } + /// + /// Gets the charset. + /// + /// + public string GetCharset() + { + // find: + string result = SearchCandidates(new Dictionary() + { + { "charset", "charset" } + }, true); + + if (String.IsNullOrEmpty(result)) + { + // find: + result = SearchCandidates(new Dictionary() + { + { "http-equiv|Content-Type", "content" } + }); + + int charsetStart = result.IndexOf("charset="); + if (charsetStart > 0) + { + charsetStart += 8; + result = result.Substring(charsetStart, result.Length - charsetStart); + } + } + + return String.IsNullOrEmpty(result) ? null : result.ToUpper(); + } + + /// /// Searches the candidates. /// /// The candidates. /// - private string SearchCandidates(Dictionary candidates) + private string SearchCandidates(Dictionary candidates, bool simple = false) { string result = null; foreach (var candidate in candidates) { + XElement element; string[] type = candidate.Key.Split('|'); - XElement element = Tags - .Where(item => String.Equals(item.GetAttributeValue(type[0], null), type[1], StringComparison.OrdinalIgnoreCase)) - .FirstOrDefault(); + if (simple) + { + element = Tags + .Where(item => item.GetAttributeValue(type[0], null) != null) + .FirstOrDefault(); + } + else + { + element = Tags + .Where(item => String.Equals(item.GetAttributeValue(type[0], null), type[1], StringComparison.OrdinalIgnoreCase)) + .FirstOrDefault(); + } if (element != null) { diff --git a/PortablePorts/NReadability/NReadabilityTranscoder.cs b/PortablePorts/NReadability/NReadabilityTranscoder.cs index a04a70d..1180aee 100644 --- a/PortablePorts/NReadability/NReadabilityTranscoder.cs +++ b/PortablePorts/NReadability/NReadabilityTranscoder.cs @@ -252,12 +252,14 @@ namespace ReadSharp.Ports.NReadability MetaExtractor metaExtractor = new MetaExtractor(transcodedXmlDocument); + string charset = null; string description = null; Uri image = null; Uri favicon = null; if (metaExtractor.HasValue) { + charset = metaExtractor.GetCharset(); description = metaExtractor.GetMetaDescription(); string imageString = metaExtractor.GetMetaImage(); string faviconString = metaExtractor.GetMetaFavicon(); @@ -283,6 +285,7 @@ namespace ReadSharp.Ports.NReadability ExtractedFavicon = favicon, ExtractedImage = image, NextPageUrl = nextPageUrl, + Charset = charset, Images = images }; } diff --git a/PortablePorts/NReadability/TranscodingResult.cs b/PortablePorts/NReadability/TranscodingResult.cs index a6bfff6..dab6206 100644 --- a/PortablePorts/NReadability/TranscodingResult.cs +++ b/PortablePorts/NReadability/TranscodingResult.cs @@ -28,6 +28,8 @@ namespace ReadSharp.Ports.NReadability public Uri ExtractedFavicon { get; set; } + public string Charset { get; set; } + public XDocument RawDocument { get; set; } public IEnumerable Images { get; set; } diff --git a/ReadSharp.Tests/ReadTests.cs b/ReadSharp.Tests/ReadTests.cs index cfe9f7d..8273480 100644 --- a/ReadSharp.Tests/ReadTests.cs +++ b/ReadSharp.Tests/ReadTests.cs @@ -73,5 +73,27 @@ namespace ReadSharp.Tests Assert.True(result.Content.Substring(0, 4) == " + diff --git a/ReadSharp/Reader.cs b/ReadSharp/Reader.cs index 4ca23e8..bd31963 100644 --- a/ReadSharp/Reader.cs +++ b/ReadSharp/Reader.cs @@ -1,10 +1,12 @@ using ReadSharp.Ports.NReadability; using System; using System.Collections.Generic; +using System.IO; using System.Linq; using System.Net; using System.Net.Http; using System.Reflection; +using System.Text; using System.Threading.Tasks; using System.Xml.Linq; @@ -25,6 +27,12 @@ namespace ReadSharp /// protected readonly HttpClient _httpClient; + /// + /// The NReadability transcoder + /// + protected NReadabilityTranscoder _transcoder; + + /// /// Initializes a new instance of the class. @@ -34,6 +42,16 @@ namespace ReadSharp /// Request timeout (in seconds). public Reader(string userAgent = null, HttpMessageHandler handler = null, int? timeout = null) { + // initialize transcoder + _transcoder = new NReadabilityTranscoder( + dontStripUnlikelys: false, + dontNormalizeSpacesInTextContent: true, + dontWeightClasses: false, + readingStyle: ReadingStyle.Ebook, + readingMargin: ReadingMargin.Narrow, + readingSize: ReadingSize.Medium + ); + // override user agent if (!string.IsNullOrEmpty(userAgent)) { @@ -58,6 +76,8 @@ namespace ReadSharp // add accepted encodings _httpClient.DefaultRequestHeaders.TryAddWithoutValidation("Accept-Encoding", "gzip,deflate"); + _httpClient.DefaultRequestHeaders.TryAddWithoutValidation("Accept-Charset", "UTF-8"); + // add user agent string version = Assembly.GetExecutingAssembly().FullName.Split(',')[1].Split('=')[1]; _httpClient.DefaultRequestHeaders.TryAddWithoutValidation("User-Agent", String.Format(_userAgent, "; ReadSharp/" + version)); @@ -77,37 +97,23 @@ namespace ReadSharp /// public async Task
Read(Uri uri, bool bodyOnly = true, bool noHeadline = false) { - // initialize transcoder - NReadabilityTranscoder transcoder = new NReadabilityTranscoder( - dontStripUnlikelys: false, - dontNormalizeSpacesInTextContent: true, - dontWeightClasses: false, - readingStyle: ReadingStyle.Ebook, - readingMargin: ReadingMargin.Narrow, - readingSize: ReadingSize.Medium - ); - // get HTML string from URI - string htmlResponse = await Request(uri); + Response response = await Request(uri); - // set properties for processing - TranscodingInput transcodingInput = new TranscodingInput(htmlResponse) + // readability + TranscodingResult transcodingResult = ExtractReadableInformation(uri, response.Stream, bodyOnly, noHeadline); + + Encoding encoding = GetEncodingFromString(transcodingResult.Charset); + + + // extract again if encoding didn't match or failed to retrieve + if (encoding != null && ( + String.IsNullOrEmpty(response.Charset) + || + !String.Equals(response.Charset, transcodingResult.Charset, StringComparison.OrdinalIgnoreCase))) { - Url = uri.ToString(), - DomSerializationParams = new DomSerializationParams() - { - BodyOnly = bodyOnly, - NoHeadline = noHeadline, - PrettyPrint = true, - DontIncludeContentTypeMetaElement = true, - DontIncludeMobileSpecificMetaElements = true, - DontIncludeDocTypeMetaElement = false, - DontIncludeGeneratorMetaElement = true - } - }; - - // process/transcode HTML - TranscodingResult transcodingResult = transcoder.Transcode(transcodingInput); + transcodingResult = ExtractReadableInformation(uri, response.Stream, bodyOnly, noHeadline, encoding); + } // get images from article List images = transcodingResult.Images.Select(image => @@ -138,12 +144,55 @@ namespace ReadSharp + /// + /// Extracts the readable information. + /// + /// The URI. + /// The text stream. + /// if set to true [body only]. + /// if set to true [no headline]. + /// + protected TranscodingResult ExtractReadableInformation( + Uri uri, + Stream textStream, + bool bodyOnly = true, + bool noHeadline = false, + Encoding encoding = null) + { + + // response stream to text + textStream.Position = 0; + StreamReader streamReader = new StreamReader(textStream, encoding ?? Encoding.UTF8); + string text = streamReader.ReadToEnd(); + + // set properties for processing + TranscodingInput transcodingInput = new TranscodingInput(text) + { + Url = uri.ToString(), + DomSerializationParams = new DomSerializationParams() + { + BodyOnly = bodyOnly, + NoHeadline = noHeadline, + PrettyPrint = true, + DontIncludeContentTypeMetaElement = true, + DontIncludeMobileSpecificMetaElements = true, + DontIncludeDocTypeMetaElement = false, + DontIncludeGeneratorMetaElement = true + } + }; + + // process/transcode HTML + return _transcoder.Transcode(transcodingInput); + } + + + /// /// Fetches a resource /// /// The URI. /// - protected async Task Request(Uri uri) + private async Task Request(Uri uri) { HttpRequestMessage request = new HttpRequestMessage(HttpMethod.Get, uri); HttpResponseMessage response = null; @@ -167,7 +216,40 @@ namespace ReadSharp } // read response - return await response.Content.ReadAsStringAsync(); + Stream responseStream = await response.Content.ReadAsStreamAsync(); + + return new Response() + { + Stream = responseStream, + Charset = response.Content.Headers.ContentType.CharSet + }; + } + + + /// + /// Gets the encoding from string. + /// + /// The encoding. + /// + private Encoding GetEncodingFromString(string encoding) + { + Encoding correctEncoding; + + if (String.IsNullOrEmpty(encoding)) + { + return null; + } + + try + { + correctEncoding = Encoding.GetEncoding(encoding); + } + catch + { + return null; + } + + return correctEncoding; } } } \ No newline at end of file