move PocketSharp.Reader

2013-12-17 22:50:36 +01:00
parent de99ff8eef
commit 2fb91d3a2a
40 changed files with 12142 additions and 0 deletions
@@ -0,0 +1,89 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <Choose>
+    <When Condition="$(NCrunchOriginalSolutionDir) != '' And $(NCrunchOriginalSolutionDir) != '*Undefined*'">
+      <PropertyGroup>
+        <FodySolutionDir>$(NCrunchOriginalSolutionDir)</FodySolutionDir>
+      </PropertyGroup>
+    </When>
+    <When Condition="$(SolutionDir) != '' And $(SolutionDir) != '*Undefined*'">
+      <PropertyGroup>
+        <FodySolutionDir>$(SolutionDir)</FodySolutionDir>
+      </PropertyGroup>
+    </When>
+    <When Condition="$(SolutionDir) == '' Or $(SolutionDir) == '*Undefined*'">
+      <PropertyGroup>
+        <FodySolutionDir>$(MSBuildProjectDirectory)\..\</FodySolutionDir>
+      </PropertyGroup>
+    </When>
+  </Choose>
+  <Choose>
+    <When Condition="$(KeyOriginatorFile) != '' And $(KeyOriginatorFile) != '*Undefined*'">
+      <PropertyGroup>
+        <FodyKeyFilePath>$(KeyOriginatorFile)</FodyKeyFilePath>
+      </PropertyGroup>
+    </When>
+    <When Condition="$(AssemblyOriginatorKeyFile) != '' And $(AssemblyOriginatorKeyFile) != '*Undefined*'">
+      <PropertyGroup>
+        <FodyKeyFilePath>$(AssemblyOriginatorKeyFile)</FodyKeyFilePath>
+      </PropertyGroup>
+    </When>
+    <Otherwise >
+      <PropertyGroup>
+        <FodyKeyFilePath></FodyKeyFilePath>
+      </PropertyGroup>
+    </Otherwise>
+  </Choose>
+  <PropertyGroup>
+    <IntermediateDir>$(ProjectDir)$(IntermediateOutputPath)</IntermediateDir>
+    <FodyMessageImportance Condition="$(FodyMessageImportance) == '' Or $(FodyMessageImportance) == '*Undefined*'">Low</FodyMessageImportance>
+    <FodySignAssembly Condition="$(FodySignAssembly) == '' Or $(FodySignAssembly) == '*Undefined*'">$(SignAssembly)</FodySignAssembly>
+    <FodyPath Condition="$(FodyPath) == '' Or $(FodyPath) == '*Undefined*'">$(MSBuildThisFileDirectory)</FodyPath>
+  </PropertyGroup>
+  <UsingTask
+      TaskName="Fody.WeavingTask"
+      AssemblyFile="$(FodyPath)\Fody.dll" />
+  <Target
+      AfterTargets="AfterCompile"
+      Name="WinFodyTarget"
+      Condition=" '$(OS)' == 'Windows_NT'">
+
+    <Fody.WeavingTask
+          AssemblyPath="@(IntermediateAssembly)"
+          IntermediateDir="$(IntermediateDir)"
+          KeyFilePath="$(FodyKeyFilePath)"
+          MessageImportance="$(FodyMessageImportance)"
+          ProjectDirectory="$(ProjectDir)"
+          SolutionDir="$(FodySolutionDir)"
+          References="@(ReferencePath)"
+          SignAssembly="$(FodySignAssembly)"
+          ReferenceCopyLocalPaths="@(ReferenceCopyLocalPaths)"
+          DefineConstants="$(DefineConstants)"
+      />
+  </Target>
+
+  <Target
+      AfterTargets="AfterBuild"
+      Name="NonWinFodyTarget"
+      Condition=" '$(OS)' != 'Windows_NT'">
+    <Fody.WeavingTask
+          AssemblyPath="$(TargetPath)"
+          IntermediateDir="$(IntermediateDir)"
+          KeyFilePath="$(FodyKeyFilePath)"
+          MessageImportance="$(FodyMessageImportance)"
+          ProjectDirectory="$(ProjectDir)"
+          SolutionDir="$(FodySolutionDir)"
+          References="@(ReferencePath)"
+          SignAssembly="$(FodySignAssembly)"
+          ReferenceCopyLocalPaths="$(ReferenceCopyLocalPaths)"
+          DefineConstants="$(DefineConstants)"
+      />
+  </Target>
+
+
+  <!--Support for ncrunch-->
+  <ItemGroup>
+    <None Include="$(FodyPath)\*.*" />
+  </ItemGroup>
+
+</Project>
@@ -0,0 +1,4 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Weavers>
+  <PropertyChanged />
+</Weavers>
@@ -0,0 +1,23 @@
+using ReadSharp.Models;
+using System;
+using System.Threading.Tasks;
+
+namespace ReadSharp
+{
+  public interface IPocketReader
+  {
+    /// <summary>
+    /// Reads article content from the given PocketItem.
+    /// This method does not use the official Article View API, which is private.
+    /// The PocketReader is based on a custom PCL port of NReadability and SgmlReader.
+    /// </summary>
+    /// <param name="uri">An URI to extract the content from.</param>
+    /// <param name="bodyOnly">if set to <c>true</c> [only body is returned].</param>
+    /// <param name="noHeadline">if set to <c>true</c> [no headline (h1) is included].</param>
+    /// <returns>
+    /// A Pocket article with extracted content and title.
+    /// </returns>
+    /// <exception cref="Exception"></exception>
+    Task<PocketArticle> Read(Uri uri, bool bodyOnly = true, bool noHeadline = false);
+  }
+}
@@ -0,0 +1,45 @@
+using PropertyChanged;
+using System;
+using System.Collections.Generic;
+
+namespace ReadSharp.Models
+{
+  /// <summary>
+  /// Readable article
+  /// </summary>
+  [ImplementPropertyChanged]
+  public class PocketArticle
+  {
+    /// <summary>
+    /// Gets or sets the content.
+    /// </summary>
+    /// <value>
+    /// The content.
+    /// </value>
+    public string Content { get; set; }
+
+    /// <summary>
+    /// Gets or sets the images.
+    /// </summary>
+    /// <value>
+    /// The images.
+    /// </value>
+    public List<PocketArticleImage> Images { get; set; }
+
+    /// <summary>
+    /// Gets or sets the title.
+    /// </summary>
+    /// <value>
+    /// The title.
+    /// </value>
+    public string Title { get; set; }
+
+    /// <summary>
+    /// Gets or sets the next page URL.
+    /// </summary>
+    /// <value>
+    /// The next page URL.
+    /// </value>
+    public Uri NextPage { get; set; }
+  }
+}
@@ -0,0 +1,36 @@
+using PropertyChanged;
+using System;
+
+namespace ReadSharp.Models
+{
+  /// <summary>
+  /// Article image
+  /// </summary>
+  [ImplementPropertyChanged]
+  public class PocketArticleImage
+  {
+    /// <summary>
+    /// Gets or sets the URI.
+    /// </summary>
+    /// <value>
+    /// The URI.
+    /// </value>
+    public Uri Uri { get; set; }
+
+    /// <summary>
+    /// Gets or sets the title.
+    /// </summary>
+    /// <value>
+    /// The title.
+    /// </value>
+    public string Title { get; set; }
+
+    /// <summary>
+    /// Gets or sets the alternative text.
+    /// </summary>
+    /// <value>
+    /// The alternative text.
+    /// </value>
+    public string AlternativeText { get; set; }
+  }
+}
@@ -0,0 +1,171 @@
+using ReadSharp.Models;
+using ReadSharp.Ports.NReadability;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Net;
+using System.Net.Http;
+using System.Threading.Tasks;
+using System.Xml.Linq;
+
+namespace ReadSharp
+{
+  /// <summary>
+  /// PocketReader
+  /// </summary>
+  public class PocketReader : IPocketReader
+  {
+    /// <summary>
+    /// Used UserAgent for HTTP request
+    /// </summary>
+    protected string _userAgent = "Mozilla/5.0 (Windows NT 6.3; Trident/7.0; rv:11.0; ARM; Mobile; Touch{0}) like Gecko";
+
+    /// <summary>
+    /// REST client used for HTML retrieval
+    /// </summary>
+    protected readonly HttpClient _httpClient;
+
+
+    /// <summary>
+    /// Initializes a new instance of the <see cref="PocketReader" /> class.
+    /// </summary>
+    /// <param name="userAgent">Custom UserAgent string.</param>
+    /// <param name="handler">The HttpMessage handler.</param>
+    /// <param name="timeout">Request timeout (in seconds).</param>
+    public PocketReader(string userAgent = null, HttpMessageHandler handler = null, int? timeout = null)
+    {
+      // override user agent
+      if (!string.IsNullOrEmpty(userAgent))
+      {
+        _userAgent = userAgent;
+      }
+
+      // initialize HTTP client
+      _httpClient = new HttpClient(handler ?? new HttpClientHandler()
+      {
+        AutomaticDecompression = DecompressionMethods.Deflate | DecompressionMethods.GZip,
+        AllowAutoRedirect = true
+      });
+
+      if (timeout.HasValue)
+      {
+        _httpClient.Timeout = TimeSpan.FromSeconds(timeout.Value);
+      }
+
+      // add accept types
+      _httpClient.DefaultRequestHeaders.TryAddWithoutValidation("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8");
+
+      // add accepted encodings
+      _httpClient.DefaultRequestHeaders.TryAddWithoutValidation("Accept-Encoding", "gzip,deflate");
+
+      // add user agent
+      _httpClient.DefaultRequestHeaders.TryAddWithoutValidation("User-Agent", string.Format(_userAgent, "; ReadSharp/3.0"));
+    }
+
+
+
+    /// <summary>
+    /// Reads article content from the given PocketItem.
+    /// This method does not use the official Article View API, which is private.
+    /// The PocketReader is based on a custom PCL port of NReadability and SgmlReader.
+    /// </summary>
+    /// <param name="uri">An URI to extract the content from.</param>
+    /// <param name="bodyOnly">if set to <c>true</c> [only body is returned].</param>
+    /// <param name="noHeadline">if set to <c>true</c> [no headline (h1) is included].</param>
+    /// <returns>
+    /// A Pocket article with extracted content and title.
+    /// </returns>
+    /// <exception cref="Exception"></exception>
+    public async Task<PocketArticle> Read(Uri uri, bool bodyOnly = true, bool noHeadline = false)
+    {
+      // initialize transcoder
+      NReadabilityTranscoder transcoder = new NReadabilityTranscoder(
+        dontStripUnlikelys: false,
+        dontNormalizeSpacesInTextContent: true,
+        dontWeightClasses: false,
+        readingStyle: ReadingStyle.Ebook,
+        readingMargin: ReadingMargin.Narrow,
+        readingSize: ReadingSize.Medium
+      );
+
+      // get HTML string from URI
+      string htmlResponse = await Request(uri);
+
+      // set properties for processing
+      TranscodingInput transcodingInput = new TranscodingInput(htmlResponse)
+      {
+        Url = uri.ToString(),
+        DomSerializationParams = new DomSerializationParams()
+        {
+          BodyOnly = bodyOnly,
+          NoHeadline = noHeadline,
+          PrettyPrint = true,
+          DontIncludeContentTypeMetaElement = true,
+          DontIncludeMobileSpecificMetaElements = true,
+          DontIncludeDocTypeMetaElement = false,
+          DontIncludeGeneratorMetaElement = true
+        }
+      };
+
+      // process/transcode HTML
+      TranscodingResult transcodingResult = transcoder.Transcode(transcodingInput);
+
+      // get images from article
+      List<PocketArticleImage> images = transcodingResult.Images.Select<XElement, PocketArticleImage>(image =>
+      {
+        Uri imageUri;
+        Uri.TryCreate(image.GetAttributeValue("src", null), UriKind.Absolute, out imageUri);
+
+        return new PocketArticleImage()
+        {
+          Uri = imageUri,
+          Title = image.GetAttributeValue("title", null),
+          AlternativeText = image.GetAttributeValue("alt", null)
+        };
+      }).ToList();
+
+      // create article
+      return new PocketArticle()
+      {
+        Content = transcodingResult.ExtractedContent,
+        Images = images,
+        Title = transcodingResult.ExtractedTitle,
+        NextPage = transcodingResult.NextPageUrl != null ? new Uri(transcodingResult.NextPageUrl, UriKind.Absolute) : null
+      };
+    }
+
+
+
+    /// <summary>
+    /// Fetches a resource
+    /// </summary>
+    /// <param name="uri">The URI.</param>
+    /// <returns></returns>
+    protected async Task<string> Request(Uri uri)
+    {
+      HttpRequestMessage request = new HttpRequestMessage(HttpMethod.Get, uri);
+      HttpResponseMessage response = null;
+
+      // make async request
+      try
+      {
+        response = await _httpClient.SendAsync(request);
+      }
+      catch (HttpRequestException exc)
+      {
+        throw new Exception(exc.Message, exc);
+      }
+
+      // validate HTTP response
+      if (response.StatusCode != HttpStatusCode.OK)
+      {
+        string exceptionString = String.Format("Request error: {0} ({1})", response.ReasonPhrase, (int)response.StatusCode);
+
+        throw new Exception(exceptionString);
+      }
+
+      // read response
+      return await response.Content.ReadAsStringAsync();
+    }
+  }
+}
@@ -14,6 +14,7 @@
    <TargetFrameworkProfile>Profile96</TargetFrameworkProfile>
    <FileAlignment>512</FileAlignment>
    <ProjectTypeGuids>{786C830F-07A1-408B-BD7F-6EE04809D6DB};{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}</ProjectTypeGuids>
+    <FodyPath>..\packages\Fody.1.19.1.0</FodyPath>
  </PropertyGroup>
  <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
    <DebugSymbols>true</DebugSymbols>
@@ -34,11 +35,62 @@
  </PropertyGroup>
  <ItemGroup>
    <!-- A reference to the entire .NET Framework is automatically included -->
+    <None Include="Fody.targets" />
+    <None Include="packages.config" />
  </ItemGroup>
  <ItemGroup>
+    <Compile Include="IPocketReader.cs" />
+    <Compile Include="Models\PocketArticle.cs" />
+    <Compile Include="Models\PocketArticleImage.cs" />
+    <Compile Include="PocketReader.cs" />
    <Compile Include="Properties\AssemblyInfo.cs" />
  </ItemGroup>
+  <ItemGroup>
+    <Reference Include="Microsoft.Threading.Tasks">
+      <HintPath>..\packages\Microsoft.Bcl.Async.1.0.165\lib\portable-net40+sl4+win8+wp71\Microsoft.Threading.Tasks.dll</HintPath>
+    </Reference>
+    <Reference Include="Microsoft.Threading.Tasks.Extensions">
+      <HintPath>..\packages\Microsoft.Bcl.Async.1.0.165\lib\portable-net40+sl4+win8+wp71\Microsoft.Threading.Tasks.Extensions.dll</HintPath>
+    </Reference>
+    <Reference Include="PropertyChanged">
+      <HintPath>..\packages\PropertyChanged.Fody.1.43.1.0\Lib\portable-net4+sl4+wp7+win8+MonoAndroid16+MonoTouch40\PropertyChanged.dll</HintPath>
+      <Private>False</Private>
+    </Reference>
+    <Reference Include="System.IO">
+      <HintPath>..\packages\Microsoft.Bcl.1.1.6\lib\portable-net40+sl4+win8+wp71\System.IO.dll</HintPath>
+    </Reference>
+    <Reference Include="System.Net.Http">
+      <HintPath>..\packages\Microsoft.Net.Http.2.2.18\lib\portable-net40+sl4+win8+wp71\System.Net.Http.dll</HintPath>
+    </Reference>
+    <Reference Include="System.Net.Http.Extensions">
+      <HintPath>..\packages\Microsoft.Net.Http.2.2.18\lib\portable-net40+sl4+win8+wp71\System.Net.Http.Extensions.dll</HintPath>
+    </Reference>
+    <Reference Include="System.Net.Http.Primitives">
+      <HintPath>..\packages\Microsoft.Net.Http.2.2.18\lib\portable-net40+sl4+win8+wp71\System.Net.Http.Primitives.dll</HintPath>
+    </Reference>
+    <Reference Include="System.Runtime">
+      <HintPath>..\packages\Microsoft.Bcl.1.1.6\lib\portable-net40+sl4+win8+wp71\System.Runtime.dll</HintPath>
+    </Reference>
+    <Reference Include="System.Threading.Tasks">
+      <HintPath>..\packages\Microsoft.Bcl.1.1.6\lib\portable-net40+sl4+win8+wp71\System.Threading.Tasks.dll</HintPath>
+    </Reference>
+  </ItemGroup>
+  <ItemGroup>
+    <Content Include="FodyWeavers.xml" />
+  </ItemGroup>
+  <ItemGroup>
+    <ProjectReference Include="..\PortablePorts\NReadability\NReadability.csproj">
+      <Project>{14c3ee6a-54a4-4a37-8b56-d52a3802f1c2}</Project>
+      <Name>NReadability</Name>
+    </ProjectReference>
+  </ItemGroup>
  <Import Project="$(MSBuildExtensionsPath32)\Microsoft\Portable\$(TargetFrameworkVersion)\Microsoft.Portable.CSharp.targets" />
+  <Import Project="..\packages\Microsoft.Bcl.Build.1.0.13\tools\Microsoft.Bcl.Build.targets" Condition="Exists('..\packages\Microsoft.Bcl.Build.1.0.13\tools\Microsoft.Bcl.Build.targets')" />
+  <Target Name="EnsureBclBuildImported" BeforeTargets="BeforeBuild" Condition="'$(BclBuildImported)' == ''">
+    <Error Condition="!Exists('..\packages\Microsoft.Bcl.Build.1.0.13\tools\Microsoft.Bcl.Build.targets')" Text="This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them.  For more information, see http://go.microsoft.com/fwlink/?LinkID=317567." HelpKeyword="BCLBUILD2001" />
+    <Error Condition="Exists('..\packages\Microsoft.Bcl.Build.1.0.13\tools\Microsoft.Bcl.Build.targets')" Text="The build restored NuGet packages. Build the project again to include these packages in the build. For more information, see http://go.microsoft.com/fwlink/?LinkID=317568." HelpKeyword="BCLBUILD2002" />
+  </Target>
+  <Import Project="Fody.targets" />
  <!-- To modify your build process, add your task inside one of the targets below and uncomment it. 
       Other similar extension points exist, see Microsoft.Common.targets.
  <Target Name="BeforeBuild">
@@ -0,0 +1,9 @@
+<?xml version="1.0" encoding="utf-8"?>
+<packages>
+  <package id="Fody" version="1.19.1.0" targetFramework="portable-win+net403+sl40+wp71" developmentDependency="true" />
+  <package id="Microsoft.Bcl" version="1.1.6" targetFramework="portable-win+net403+sl40+wp71" />
+  <package id="Microsoft.Bcl.Async" version="1.0.165" targetFramework="portable-win+net403+sl40+wp71" />
+  <package id="Microsoft.Bcl.Build" version="1.0.13" targetFramework="portable-win+net403+sl40+wp71" />
+  <package id="Microsoft.Net.Http" version="2.2.18" targetFramework="portable-win+net403+sl40+wp71" />
+  <package id="PropertyChanged.Fody" version="1.43.1.0" targetFramework="portable-win+net403+sl40+wp71" developmentDependency="true" />
+</packages>