using PocketSharp.Models;
using PocketSharp.Ports.NReadability;
using System;
using System.Net;
using System.Net.Http;
using System.Threading.Tasks;
namespace PocketSharp
{
///
/// PocketReader
///
public class PocketReader : IPocketReader
{
///
/// Used UserAgent for HTTP request
///
protected string _userAgent = "Mozilla/5.0 (Windows NT 6.3; Trident/7.0; rv:11.0; ARM; Mobile; Touch{0}) like Gecko";
///
/// REST client used for HTML retrieval
///
protected readonly HttpClient _httpClient;
///
/// Initializes a new instance of the class.
///
/// Custom UserAgent string.
/// The HttpMessage handler.
/// Request timeout (in seconds).
public PocketReader(string userAgent = null, HttpMessageHandler handler = null, int? timeout = null)
{
// override user agent
if (!string.IsNullOrEmpty(userAgent))
{
_userAgent = userAgent;
}
// initialize HTTP client
_httpClient = new HttpClient(handler ?? new HttpClientHandler()
{
AutomaticDecompression = DecompressionMethods.Deflate | DecompressionMethods.GZip,
AllowAutoRedirect = true
});
if (timeout.HasValue)
{
_httpClient.Timeout = TimeSpan.FromSeconds(timeout.Value);
}
// add accept types
_httpClient.DefaultRequestHeaders.TryAddWithoutValidation("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8");
// add accepted encodings
_httpClient.DefaultRequestHeaders.TryAddWithoutValidation("Accept-Encoding", "gzip,deflate");
// add user agent
_httpClient.DefaultRequestHeaders.TryAddWithoutValidation("User-Agent", string.Format(_userAgent, "; PocketSharp/3.0"));
}
///
/// Reads article content from the given URI.
/// This method does not use the official Article View API, which is private.
/// The PocketReader is based on a custom PCL port of NReadability and SgmlReader.
///
/// An URI.
/// if set to true [only body is returned].
/// if set to true [no headline (h1) is included].
///
/// A Pocket article with extracted content and title.
///
public async Task Read(Uri uri, bool bodyOnly = true, bool noHeadline = false)
{
return await Read(new PocketItem()
{
ID = null,
Uri = uri
}, bodyOnly, noHeadline);
}
///
/// Reads article content from the given PocketItem.
/// This method does not use the official Article View API, which is private.
/// The PocketReader is based on a custom PCL port of NReadability and SgmlReader.
///
/// The pocket item.
/// if set to true [only body is returned].
/// if set to true [no headline (h1) is included].
///
/// A Pocket article with extracted content and title.
///
///
public async Task Read(PocketItem item, bool bodyOnly = true, bool noHeadline = false)
{
// initialize transcoder
NReadabilityTranscoder transcoder = new NReadabilityTranscoder(
dontStripUnlikelys: false,
dontNormalizeSpacesInTextContent: true,
dontWeightClasses: false,
readingStyle: ReadingStyle.Ebook,
readingMargin: ReadingMargin.Narrow,
readingSize: ReadingSize.Medium
);
// get HTML string from URI
string htmlResponse = await Request(item.Uri);
// set properties for processing
TranscodingInput transcodingInput = new TranscodingInput(htmlResponse)
{
Url = item.Uri.ToString(),
DomSerializationParams = new DomSerializationParams()
{
BodyOnly = bodyOnly,
NoHeadline = noHeadline,
PrettyPrint = true,
DontIncludeContentTypeMetaElement = true,
DontIncludeMobileSpecificMetaElements = true,
DontIncludeDocTypeMetaElement = false,
DontIncludeGeneratorMetaElement = true
}
};
// process/transcode HTML
TranscodingResult transcodingResult = transcoder.Transcode(transcodingInput);
return new PocketArticle()
{
Content = transcodingResult.ExtractedContent,
Title = transcodingResult.ExtractedTitle,
NextPage = transcodingResult.NextPageUrl != null ? new Uri(transcodingResult.NextPageUrl, UriKind.Absolute) : null,
PocketItemID = item.ID
};
}
///
/// Fetches a resource
///
/// The URI.
///
protected async Task Request(Uri uri)
{
HttpRequestMessage request = new HttpRequestMessage(HttpMethod.Get, uri);
HttpResponseMessage response = null;
// make async request
try
{
response = await _httpClient.SendAsync(request);
}
catch (HttpRequestException exc)
{
throw new PocketException(exc.Message, exc);
}
// validate HTTP response
if (response.StatusCode != HttpStatusCode.OK)
{
string exceptionString = String.Format("Request error: {0} ({1})", response.ReasonPhrase, (int)response.StatusCode);
throw new PocketException(exceptionString);
}
// read response
return await response.Content.ReadAsStringAsync();
}
}
}