using ReadSharp.Ports.NReadability;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net;
using System.Net.Http;
using System.Reflection;
using System.Text;
using System.Threading.Tasks;
using System.Xml.Linq;
namespace ReadSharp
{
///
/// PocketReader
///
public class Reader : IReader
{
///
/// Used UserAgent for HTTP request
///
protected string _userAgent = "Mozilla/5.0 (Windows NT 6.3; Trident/7.0; rv:11.0; ARM; Mobile; Touch{0}) like Gecko";
///
/// REST client used for HTML retrieval
///
protected readonly HttpClient _httpClient;
///
/// The encoder
///
protected readonly Encodings.Encoder _encoder;
///
/// The NReadability transcoder
///
protected NReadabilityTranscoder _transcoder;
///
/// Initializes a new instance of the class.
///
/// Custom UserAgent string.
/// The HttpMessage handler.
/// Request timeout (in seconds).
public Reader(string userAgent = null, HttpMessageHandler handler = null, int? timeout = null)
{
// initialize transcoder
_transcoder = new NReadabilityTranscoder(
dontStripUnlikelys: false,
dontNormalizeSpacesInTextContent: true,
dontWeightClasses: false,
readingStyle: ReadingStyle.Ebook,
readingMargin: ReadingMargin.Narrow,
readingSize: ReadingSize.Medium
);
// initialize custom encoder
_encoder = new Encodings.Encoder(true);
// override user agent
if (!string.IsNullOrEmpty(userAgent))
{
_userAgent = userAgent;
}
// initialize HTTP client
_httpClient = new HttpClient(handler ?? new HttpClientHandler()
{
AutomaticDecompression = DecompressionMethods.Deflate | DecompressionMethods.GZip,
AllowAutoRedirect = true
});
if (timeout.HasValue)
{
_httpClient.Timeout = TimeSpan.FromSeconds(timeout.Value);
}
// add accept types
_httpClient.DefaultRequestHeaders.TryAddWithoutValidation("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8");
// add accepted encodings
_httpClient.DefaultRequestHeaders.TryAddWithoutValidation("Accept-Encoding", "gzip,deflate");
//_httpClient.DefaultRequestHeaders.TryAddWithoutValidation("Accept-Charset", "UTF-8");
// add user agent
string version = Assembly.GetExecutingAssembly().FullName.Split(',')[1].Split('=')[1];
_httpClient.DefaultRequestHeaders.TryAddWithoutValidation("User-Agent", String.Format(_userAgent, "; ReadSharp/" + version));
}
///
/// Reads article content from the given URI.
///
/// An URI to extract the content from.
/// if set to true [only body is returned].
/// if set to true [no headline (h1) is included in generated HTML].
/// if set to true [deep links with hashes are not transformed to absolute URIs].
///
/// An article with extracted content and meta information.
///
///
public async Task Read(
Uri uri,
bool bodyOnly = true,
bool noHeadline = false,
bool useDeepLinks = false)
{
HttpRequestMessage request = new HttpRequestMessage(HttpMethod.Get, uri);
HttpResponseMessage respo = null;
// make async request
try
{
respo = await _httpClient.SendAsync(request);
}
catch (HttpRequestException exc)
{
throw new Exception(exc.Message, exc);
}
// read response
string resp = await respo.Content.ReadAsStringAsync();
// get HTML string from URI
Response response = await Request(uri);
// handle deep links
if (useDeepLinks)
{
_transcoder.AnchorHrefTranformer = ReverseDeepLinks;
}
else
{
_transcoder.AnchorHrefTranformer = null;
}
// readability
TranscodingResult transcodingResult = ExtractReadableInformation(uri, response.Stream, bodyOnly, noHeadline);
Encoding encoding = _encoder.GetEncodingFromString(transcodingResult.Charset);
// extract again if encoding didn't match or failed to retrieve
if (encoding != null && (
String.IsNullOrEmpty(response.Charset)
||
!String.Equals(response.Charset, transcodingResult.Charset, StringComparison.OrdinalIgnoreCase)))
{
transcodingResult = ExtractReadableInformation(uri, response.Stream, bodyOnly, noHeadline, encoding);
}
// get images from article
List images = transcodingResult.Images.Select(image =>
{
Uri imageUri;
Uri.TryCreate(image.GetAttributeValue("src", null), UriKind.Absolute, out imageUri);
return new ArticleImage()
{
Uri = imageUri,
Title = image.GetAttributeValue("title", null),
AlternativeText = image.GetAttributeValue("alt", null)
};
}).ToList();
// create article
return new Article()
{
Title = transcodingResult.ExtractedTitle,
Description = transcodingResult.ExtractedDescription,
Content = transcodingResult.ExtractedContent,
FrontImage = transcodingResult.ExtractedImage,
Images = images,
Favicon = transcodingResult.ExtractedFavicon,
NextPage = transcodingResult.NextPageUrl != null ? new Uri(transcodingResult.NextPageUrl, UriKind.Absolute) : null,
Encoding = _encoder.GetEncodingFromString(response.Charset) ?? encoding ?? null
};
}
///
/// Extracts the readable information.
///
/// The URI.
/// The text stream.
/// if set to true [body only].
/// if set to true [no headline].
///
protected TranscodingResult ExtractReadableInformation(
Uri uri,
Stream textStream,
bool bodyOnly = true,
bool noHeadline = false,
Encoding encoding = null)
{
// response stream to text
textStream.Position = 0;
StreamReader streamReader = new StreamReader(textStream, encoding ?? Encoding.UTF8);
string text = streamReader.ReadToEnd();
// set properties for processing
TranscodingInput transcodingInput = new TranscodingInput(text)
{
Url = uri.ToString(),
DomSerializationParams = new DomSerializationParams()
{
BodyOnly = bodyOnly,
NoHeadline = noHeadline,
PrettyPrint = true,
DontIncludeContentTypeMetaElement = true,
DontIncludeMobileSpecificMetaElements = true,
DontIncludeDocTypeMetaElement = false,
DontIncludeGeneratorMetaElement = true
}
};
// process/transcode HTML
return _transcoder.Transcode(transcodingInput);
}
///
/// Reverses the deep links.
///
/// The input.
///
private AttributeTransformationResult ReverseDeepLinks(AttributeTransformationInput input)
{
string articleUrl = input.ArticleUrl;
string link = input.AttributeValue;
// remove deep-link if in article URI
if (articleUrl.Contains("#"))
{
articleUrl = articleUrl.Split('#')[0];
}
// anchor is a deep-link
if (
input.AttributeValue.Contains(articleUrl) &&
input.AttributeValue.Contains("#") &&
input.AttributeValue.Split('#')[1].Length > 0)
{
link = "#" + input.AttributeValue.Split('#')[1];
}
return new AttributeTransformationResult()
{
TransformedValue = link
};
}
///
/// Fetches a resource
///
/// The URI.
///
private async Task Request(Uri uri)
{
HttpRequestMessage request = new HttpRequestMessage(HttpMethod.Get, uri);
HttpResponseMessage response = null;
// make async request
try
{
response = await _httpClient.SendAsync(request);
}
catch (HttpRequestException exc)
{
throw new Exception(exc.Message, exc);
}
// validate HTTP response
if (response.StatusCode != HttpStatusCode.OK)
{
string exceptionString = String.Format("Request error: {0} ({1})", response.ReasonPhrase, (int)response.StatusCode);
throw new Exception(exceptionString);
}
// read response
Stream responseStream = await response.Content.ReadAsStreamAsync();
return new Response()
{
Stream = responseStream,
Charset = response.Content.Headers.ContentType.CharSet
};
}
}
}