using ReadSharp.Ports.NReadability; using System; using System.Collections.Generic; using System.IO; using System.Linq; using System.Net; using System.Net.Http; using System.Reflection; using System.Text; using System.Threading.Tasks; using System.Xml.Linq; namespace ReadSharp { /// /// PocketReader /// public class Reader : IReader { /// /// Used UserAgent for HTTP request /// protected string _userAgent = "Mozilla/5.0 (Windows NT 6.3; Trident/7.0; rv:11.0; ARM; Mobile; Touch{0}) like Gecko"; /// /// REST client used for HTML retrieval /// protected readonly HttpClient _httpClient; /// /// The encoder /// protected readonly Encodings.Encoder _encoder; /// /// The NReadability transcoder /// protected NReadabilityTranscoder _transcoder; /// /// Initializes a new instance of the class. /// /// Custom UserAgent string. /// The HttpMessage handler. /// Request timeout (in seconds). public Reader(string userAgent = null, HttpMessageHandler handler = null, int? timeout = null) { // initialize transcoder _transcoder = new NReadabilityTranscoder( dontStripUnlikelys: false, dontNormalizeSpacesInTextContent: true, dontWeightClasses: false, readingStyle: ReadingStyle.Ebook, readingMargin: ReadingMargin.Narrow, readingSize: ReadingSize.Medium ); // initialize custom encoder _encoder = new Encodings.Encoder(true); // override user agent if (!string.IsNullOrEmpty(userAgent)) { _userAgent = userAgent; } // initialize HTTP client _httpClient = new HttpClient(handler ?? new HttpClientHandler() { AutomaticDecompression = DecompressionMethods.Deflate | DecompressionMethods.GZip, AllowAutoRedirect = true }); if (timeout.HasValue) { _httpClient.Timeout = TimeSpan.FromSeconds(timeout.Value); } // add accept types _httpClient.DefaultRequestHeaders.TryAddWithoutValidation("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8"); // add accepted encodings _httpClient.DefaultRequestHeaders.TryAddWithoutValidation("Accept-Encoding", "gzip,deflate"); //_httpClient.DefaultRequestHeaders.TryAddWithoutValidation("Accept-Charset", "UTF-8"); // add user agent string version = Assembly.GetExecutingAssembly().FullName.Split(',')[1].Split('=')[1]; _httpClient.DefaultRequestHeaders.TryAddWithoutValidation("User-Agent", String.Format(_userAgent, "; ReadSharp/" + version)); } /// /// Reads article content from the given URI. /// /// An URI to extract the content from. /// if set to true [only body is returned]. /// if set to true [no headline (h1) is included in generated HTML]. /// if set to true [deep links with hashes are not transformed to absolute URIs]. /// /// An article with extracted content and meta information. /// /// public async Task
Read( Uri uri, bool bodyOnly = true, bool noHeadline = false, bool useDeepLinks = false) { HttpRequestMessage request = new HttpRequestMessage(HttpMethod.Get, uri); HttpResponseMessage respo = null; // make async request try { respo = await _httpClient.SendAsync(request); } catch (HttpRequestException exc) { throw new Exception(exc.Message, exc); } // read response string resp = await respo.Content.ReadAsStringAsync(); // get HTML string from URI Response response = await Request(uri); // handle deep links if (useDeepLinks) { _transcoder.AnchorHrefTranformer = ReverseDeepLinks; } else { _transcoder.AnchorHrefTranformer = null; } // readability TranscodingResult transcodingResult = ExtractReadableInformation(uri, response.Stream, bodyOnly, noHeadline); Encoding encoding = _encoder.GetEncodingFromString(transcodingResult.Charset); // extract again if encoding didn't match or failed to retrieve if (encoding != null && ( String.IsNullOrEmpty(response.Charset) || !String.Equals(response.Charset, transcodingResult.Charset, StringComparison.OrdinalIgnoreCase))) { transcodingResult = ExtractReadableInformation(uri, response.Stream, bodyOnly, noHeadline, encoding); } // get images from article List images = transcodingResult.Images.Select(image => { Uri imageUri; Uri.TryCreate(image.GetAttributeValue("src", null), UriKind.Absolute, out imageUri); return new ArticleImage() { Uri = imageUri, Title = image.GetAttributeValue("title", null), AlternativeText = image.GetAttributeValue("alt", null) }; }).ToList(); // create article return new Article() { Title = transcodingResult.ExtractedTitle, Description = transcodingResult.ExtractedDescription, Content = transcodingResult.ExtractedContent, FrontImage = transcodingResult.ExtractedImage, Images = images, Favicon = transcodingResult.ExtractedFavicon, NextPage = transcodingResult.NextPageUrl != null ? new Uri(transcodingResult.NextPageUrl, UriKind.Absolute) : null, Encoding = _encoder.GetEncodingFromString(response.Charset) ?? encoding ?? null }; } /// /// Extracts the readable information. /// /// The URI. /// The text stream. /// if set to true [body only]. /// if set to true [no headline]. /// protected TranscodingResult ExtractReadableInformation( Uri uri, Stream textStream, bool bodyOnly = true, bool noHeadline = false, Encoding encoding = null) { // response stream to text textStream.Position = 0; StreamReader streamReader = new StreamReader(textStream, encoding ?? Encoding.UTF8); string text = streamReader.ReadToEnd(); // set properties for processing TranscodingInput transcodingInput = new TranscodingInput(text) { Url = uri.ToString(), DomSerializationParams = new DomSerializationParams() { BodyOnly = bodyOnly, NoHeadline = noHeadline, PrettyPrint = true, DontIncludeContentTypeMetaElement = true, DontIncludeMobileSpecificMetaElements = true, DontIncludeDocTypeMetaElement = false, DontIncludeGeneratorMetaElement = true } }; // process/transcode HTML return _transcoder.Transcode(transcodingInput); } /// /// Reverses the deep links. /// /// The input. /// private AttributeTransformationResult ReverseDeepLinks(AttributeTransformationInput input) { string articleUrl = input.ArticleUrl; string link = input.AttributeValue; // remove deep-link if in article URI if (articleUrl.Contains("#")) { articleUrl = articleUrl.Split('#')[0]; } // anchor is a deep-link if ( input.AttributeValue.Contains(articleUrl) && input.AttributeValue.Contains("#") && input.AttributeValue.Split('#')[1].Length > 0) { link = "#" + input.AttributeValue.Split('#')[1]; } return new AttributeTransformationResult() { TransformedValue = link }; } /// /// Fetches a resource /// /// The URI. /// private async Task Request(Uri uri) { HttpRequestMessage request = new HttpRequestMessage(HttpMethod.Get, uri); HttpResponseMessage response = null; // make async request try { response = await _httpClient.SendAsync(request); } catch (HttpRequestException exc) { throw new Exception(exc.Message, exc); } // validate HTTP response if (response.StatusCode != HttpStatusCode.OK) { string exceptionString = String.Format("Request error: {0} ({1})", response.ReasonPhrase, (int)response.StatusCode); throw new Exception(exceptionString); } // read response Stream responseStream = await response.Content.ReadAsStreamAsync(); return new Response() { Stream = responseStream, Charset = response.Content.Headers.ContentType.CharSet }; } } }