fix charset bugs

2013-12-19 21:13:23 +01:00
parent ec8624d1e2
commit cebb186db6
8 changed files with 199 additions and 36 deletions
@@ -103,22 +103,64 @@ namespace ReadSharp.Ports.NReadability
    }


+    /// <summary>
+    /// Gets the charset.
+    /// </summary>
+    /// <returns></returns>
+    public string GetCharset()
+    {
+      // find: <meta charset="utf-8">
+      string result = SearchCandidates(new Dictionary<string, string>()
+      {
+        { "charset", "charset" }
+      }, true);
+
+      if (String.IsNullOrEmpty(result))
+      {
+        // find: <meta http-equiv="Content-Type" content="text/html; charset=windows-1250">
+        result = SearchCandidates(new Dictionary<string, string>()
+        {
+          { "http-equiv|Content-Type", "content" }
+        });
+
+        int charsetStart = result.IndexOf("charset=");
+        if (charsetStart > 0)
+        {
+          charsetStart += 8;
+          result = result.Substring(charsetStart, result.Length - charsetStart);
+        }
+      }
+
+      return String.IsNullOrEmpty(result) ? null : result.ToUpper();
+    }
+
+
    /// <summary>
    /// Searches the candidates.
    /// </summary>
    /// <param name="candidates">The candidates.</param>
    /// <returns></returns>
-    private string SearchCandidates(Dictionary<string, string> candidates)
+    private string SearchCandidates(Dictionary<string, string> candidates, bool simple = false)
    {
      string result = null;

      foreach (var candidate in candidates)
      {
+        XElement element;
        string[] type = candidate.Key.Split('|');

-        XElement element = Tags
-          .Where(item => String.Equals(item.GetAttributeValue(type[0], null), type[1], StringComparison.OrdinalIgnoreCase))
-          .FirstOrDefault();
+        if (simple)
+        {
+          element = Tags
+            .Where(item => item.GetAttributeValue(type[0], null) != null)
+            .FirstOrDefault();
+        }
+        else
+        {
+          element = Tags
+            .Where(item => String.Equals(item.GetAttributeValue(type[0], null), type[1], StringComparison.OrdinalIgnoreCase))
+            .FirstOrDefault();
+        }

        if (element != null)
        {
@@ -252,12 +252,14 @@ namespace ReadSharp.Ports.NReadability

      MetaExtractor metaExtractor = new MetaExtractor(transcodedXmlDocument);

+      string charset = null;
      string description = null;
      Uri image = null;
      Uri favicon = null;

      if (metaExtractor.HasValue)
      {
+        charset = metaExtractor.GetCharset();
        description = metaExtractor.GetMetaDescription();
        string imageString = metaExtractor.GetMetaImage();
        string faviconString = metaExtractor.GetMetaFavicon();
@@ -283,6 +285,7 @@ namespace ReadSharp.Ports.NReadability
            ExtractedFavicon = favicon,
            ExtractedImage = image,
            NextPageUrl = nextPageUrl,
+            Charset = charset,
            Images = images
          };
    }
@@ -28,6 +28,8 @@ namespace ReadSharp.Ports.NReadability

    public Uri ExtractedFavicon { get; set; }

+    public string Charset { get; set; }
+
    public XDocument RawDocument { get; set; }

    public IEnumerable<XElement> Images { get; set; }
@@ -73,5 +73,27 @@ namespace ReadSharp.Tests

      Assert.True(result.Content.Substring(0, 4) == "<div");
    }
+
+
+    [Fact]
+    public async Task TestCzechCharsets()
+    {
+      string expectedTitle = "Kouzelné české Vánoce";
+      Article result = await reader.Read(new Uri("http://www.czech.cz/cz/Zivot-a-prace/Jak-se-zije-v-CR/Zvyky-a-tradice/Kouzelne-ceske-Vanoce"));
+      Assert.Equal(result.Title, expectedTitle);
+
+      expectedTitle = "Kolik se dá vydělat na volné noze?";
+      result = await reader.Read(new Uri("http://navolnenoze.cz/blog/vydelky/"));
+      Assert.Equal(result.Title, expectedTitle);
+    }
+
+
+    [Fact]
+    public async Task TestDifferentCharsets()
+    {
+      string expectedTitle = "优艺客-专注互联网品牌建设-原韩雪冬网页设计工作室（公司站）";
+      Article result = await reader.Read(new Uri("http://www.uelike.com"));
+      Assert.Equal(result.Title, expectedTitle);
+    }
  }
 }
@@ -0,0 +1,11 @@
+using System.IO;
+
+namespace ReadSharp
+{
+  internal class Response
+  {
+    public Stream Stream { get; set; }
+
+    public string Charset { get; set; }
+  }
+}
@@ -22,5 +22,5 @@
 // You can specify all the values or you can default the Build and Revision Numbers
 // by using the '*' as shown below:
 // [assembly: AssemblyVersion("1.0.*")]
-[assembly: AssemblyVersion("4.0.0")]
-[assembly: AssemblyFileVersion("4.0.0")]
+[assembly: AssemblyVersion("4.1.0")]
+[assembly: AssemblyFileVersion("4.1.0")]
@@ -44,6 +44,7 @@
    <Compile Include="Models\ArticleImage.cs" />
    <Compile Include="Reader.cs" />
    <Compile Include="Properties\AssemblyInfo.cs" />
+    <Compile Include="Models\Response.cs" />
  </ItemGroup>
  <ItemGroup>
    <Reference Include="Microsoft.Threading.Tasks">
@@ -1,10 +1,12 @@
 using ReadSharp.Ports.NReadability;
 using System;
 using System.Collections.Generic;
+using System.IO;
 using System.Linq;
 using System.Net;
 using System.Net.Http;
 using System.Reflection;
+using System.Text;
 using System.Threading.Tasks;
 using System.Xml.Linq;

@@ -25,6 +27,12 @@ namespace ReadSharp
    /// </summary>
    protected readonly HttpClient _httpClient;

+    /// <summary>
+    /// The NReadability transcoder
+    /// </summary>
+    protected NReadabilityTranscoder _transcoder;
+
+

    /// <summary>
    /// Initializes a new instance of the <see cref="Reader" /> class.
@@ -34,6 +42,16 @@ namespace ReadSharp
    /// <param name="timeout">Request timeout (in seconds).</param>
    public Reader(string userAgent = null, HttpMessageHandler handler = null, int? timeout = null)
    {
+      // initialize transcoder
+      _transcoder = new NReadabilityTranscoder(
+        dontStripUnlikelys: false,
+        dontNormalizeSpacesInTextContent: true,
+        dontWeightClasses: false,
+        readingStyle: ReadingStyle.Ebook,
+        readingMargin: ReadingMargin.Narrow,
+        readingSize: ReadingSize.Medium
+      );
+
      // override user agent
      if (!string.IsNullOrEmpty(userAgent))
      {
@@ -58,6 +76,8 @@ namespace ReadSharp
      // add accepted encodings
      _httpClient.DefaultRequestHeaders.TryAddWithoutValidation("Accept-Encoding", "gzip,deflate");

+      _httpClient.DefaultRequestHeaders.TryAddWithoutValidation("Accept-Charset", "UTF-8");
+
      // add user agent
      string version = Assembly.GetExecutingAssembly().FullName.Split(',')[1].Split('=')[1];
      _httpClient.DefaultRequestHeaders.TryAddWithoutValidation("User-Agent", String.Format(_userAgent, "; ReadSharp/" + version));
@@ -77,37 +97,23 @@ namespace ReadSharp
    /// <exception cref="Exception"></exception>
    public async Task<Article> Read(Uri uri, bool bodyOnly = true, bool noHeadline = false)
    {
-      // initialize transcoder
-      NReadabilityTranscoder transcoder = new NReadabilityTranscoder(
-        dontStripUnlikelys: false,
-        dontNormalizeSpacesInTextContent: true,
-        dontWeightClasses: false,
-        readingStyle: ReadingStyle.Ebook,
-        readingMargin: ReadingMargin.Narrow,
-        readingSize: ReadingSize.Medium
-      );
-
      // get HTML string from URI
-      string htmlResponse = await Request(uri);
+      Response response = await Request(uri);

-      // set properties for processing
-      TranscodingInput transcodingInput = new TranscodingInput(htmlResponse)
+      // readability
+      TranscodingResult transcodingResult = ExtractReadableInformation(uri, response.Stream, bodyOnly, noHeadline);
+
+      Encoding encoding = GetEncodingFromString(transcodingResult.Charset);
+
+
+      // extract again if encoding didn't match or failed to retrieve
+      if (encoding != null && (
+        String.IsNullOrEmpty(response.Charset)
+        ||
+        !String.Equals(response.Charset, transcodingResult.Charset, StringComparison.OrdinalIgnoreCase)))
      {
-        Url = uri.ToString(),
-        DomSerializationParams = new DomSerializationParams()
-        {
-          BodyOnly = bodyOnly,
-          NoHeadline = noHeadline,
-          PrettyPrint = true,
-          DontIncludeContentTypeMetaElement = true,
-          DontIncludeMobileSpecificMetaElements = true,
-          DontIncludeDocTypeMetaElement = false,
-          DontIncludeGeneratorMetaElement = true
-        }
-      };
-
-      // process/transcode HTML
-      TranscodingResult transcodingResult = transcoder.Transcode(transcodingInput);
+        transcodingResult = ExtractReadableInformation(uri, response.Stream, bodyOnly, noHeadline, encoding);
+      }

      // get images from article
      List<ArticleImage> images = transcodingResult.Images.Select<XElement, ArticleImage>(image =>
@@ -138,12 +144,55 @@ namespace ReadSharp



+    /// <summary>
+    /// Extracts the readable information.
+    /// </summary>
+    /// <param name="uri">The URI.</param>
+    /// <param name="textStream">The text stream.</param>
+    /// <param name="bodyOnly">if set to <c>true</c> [body only].</param>
+    /// <param name="noHeadline">if set to <c>true</c> [no headline].</param>
+    /// <returns></returns>
+    protected TranscodingResult ExtractReadableInformation(
+      Uri uri,
+      Stream textStream,
+      bool bodyOnly = true,
+      bool noHeadline = false,
+      Encoding encoding = null)
+    {
+
+      // response stream to text
+      textStream.Position = 0;
+      StreamReader streamReader = new StreamReader(textStream, encoding ?? Encoding.UTF8);
+      string text = streamReader.ReadToEnd();
+
+      // set properties for processing
+      TranscodingInput transcodingInput = new TranscodingInput(text)
+      {
+        Url = uri.ToString(),
+        DomSerializationParams = new DomSerializationParams()
+        {
+          BodyOnly = bodyOnly,
+          NoHeadline = noHeadline,
+          PrettyPrint = true,
+          DontIncludeContentTypeMetaElement = true,
+          DontIncludeMobileSpecificMetaElements = true,
+          DontIncludeDocTypeMetaElement = false,
+          DontIncludeGeneratorMetaElement = true
+        }
+      };
+
+      // process/transcode HTML
+      return _transcoder.Transcode(transcodingInput);
+    }
+
+
+
    /// <summary>
    /// Fetches a resource
    /// </summary>
    /// <param name="uri">The URI.</param>
    /// <returns></returns>
-    protected async Task<string> Request(Uri uri)
+    private async Task<Response> Request(Uri uri)
    {
      HttpRequestMessage request = new HttpRequestMessage(HttpMethod.Get, uri);
      HttpResponseMessage response = null;
@@ -167,7 +216,40 @@ namespace ReadSharp
      }

      // read response
-      return await response.Content.ReadAsStringAsync();
+      Stream responseStream = await response.Content.ReadAsStreamAsync();
+
+      return new Response()
+      {
+        Stream = responseStream,
+        Charset = response.Content.Headers.ContentType.CharSet
+      };
+    }
+
+
+    /// <summary>
+    /// Gets the encoding from string.
+    /// </summary>
+    /// <param name="encoding">The encoding.</param>
+    /// <returns></returns>
+    private Encoding GetEncodingFromString(string encoding)
+    {
+      Encoding correctEncoding;
+
+      if (String.IsNullOrEmpty(encoding))
+      {
+        return null;
+      }
+
+      try
+      {
+        correctEncoding = Encoding.GetEncoding(encoding);
+      }
+      catch
+      {
+        return null;
+      }
+
+      return correctEncoding;
    }
  }
 }