diff --git a/PortablePorts/NReadability/AttributeTransformationInput.cs b/PortablePorts/NReadability/AttributeTransformationInput.cs new file mode 100644 index 0000000..e9097c5 --- /dev/null +++ b/PortablePorts/NReadability/AttributeTransformationInput.cs @@ -0,0 +1,11 @@ +using System.Xml.Linq; + +namespace ReadSharp.Ports.NReadability +{ + public class AttributeTransformationInput + { + public string AttributeValue { get; set; } + + public XElement Element { get; set; } + } +} diff --git a/PortablePorts/NReadability/AttributeTransformationResult.cs b/PortablePorts/NReadability/AttributeTransformationResult.cs new file mode 100644 index 0000000..5a244fd --- /dev/null +++ b/PortablePorts/NReadability/AttributeTransformationResult.cs @@ -0,0 +1,15 @@ +namespace ReadSharp.Ports.NReadability +{ + public class AttributeTransformationResult + { + /// + /// Result of the transformation. + /// + public string TransformedValue { get; set; } + + /// + /// Name of the attribute that will be used to store the original value. Can be null. + /// + public string OriginalValueAttributeName { get; set; } + } +} diff --git a/PortablePorts/NReadability/ChildNodesTraverser.cs b/PortablePorts/NReadability/ChildNodesTraverser.cs new file mode 100644 index 0000000..b274903 --- /dev/null +++ b/PortablePorts/NReadability/ChildNodesTraverser.cs @@ -0,0 +1,67 @@ +/* + * NReadability + * http://code.google.com/p/nreadability/ + * + * Copyright 2010 Marek Stój + * http://immortal.pl/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System; +using System.Xml.Linq; + +namespace ReadSharp.Ports.NReadability +{ + internal class ChildNodesTraverser + { + private readonly Action _childNodeVisitor; + + #region Constructor(s) + + public ChildNodesTraverser(Action childNodeVisitor) + { + if (childNodeVisitor == null) + { + throw new ArgumentNullException("childNodeVisitor"); + } + + _childNodeVisitor = childNodeVisitor; + } + + #endregion + + #region Public methods + + public void Traverse(XNode node) + { + if (!(node is XContainer)) + { + throw new ArgumentException("The node must be an XContainer in order to traverse its children."); + } + + var childNode = ((XContainer)node).FirstNode; + + while (childNode != null) + { + var nextChildNode = childNode.NextNode; + + _childNodeVisitor(childNode); + + childNode = nextChildNode; + } + } + + #endregion + } +} diff --git a/PortablePorts/NReadability/Consts.cs b/PortablePorts/NReadability/Consts.cs new file mode 100644 index 0000000..9094a59 --- /dev/null +++ b/PortablePorts/NReadability/Consts.cs @@ -0,0 +1,28 @@ +using System.Reflection; + +namespace ReadSharp.Ports.NReadability +{ + public static class Consts + { + private static readonly string _nReadabilityFullName; + + #region Constructor(s) + + static Consts() + { + _nReadabilityFullName = string.Format("NReadability {0}", Assembly.GetExecutingAssembly().FullName); + } + + #endregion + + #region Properties + + public static string NReadabilityFullName + { + get { return _nReadabilityFullName; } + } + + #endregion + + } +} diff --git a/PortablePorts/NReadability/DomExtensions.cs b/PortablePorts/NReadability/DomExtensions.cs new file mode 100644 index 0000000..2efe09b --- /dev/null +++ b/PortablePorts/NReadability/DomExtensions.cs @@ -0,0 +1,303 @@ +/* + * NReadability + * http://code.google.com/p/nreadability/ + * + * Copyright 2010 Marek Stój + * http://immortal.pl/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Xml.Linq; + +namespace ReadSharp.Ports.NReadability +{ + public static class DomExtensions + { + #region XDocument extensions + + public static XElement GetBody(this XDocument document) + { + if (document == null) + { + throw new ArgumentNullException("document"); + } + + var documentRoot = document.Root; + + if (documentRoot == null) + { + return null; + } + + return documentRoot.GetElementsByTagName("body").FirstOrDefault(); + } + + public static string GetTitle(this XDocument document) + { + if (document == null) + { + throw new ArgumentNullException("document"); + } + + var documentRoot = document.Root; + + if (documentRoot == null) + { + return null; + } + + var headElement = documentRoot.GetElementsByTagName("head").FirstOrDefault(); + + if (headElement == null) + { + return ""; + } + + var titleElement = headElement.GetChildrenByTagName("title").FirstOrDefault(); + + if (titleElement == null) + { + return ""; + } + + return (titleElement.Value ?? "").Trim(); + } + + public static XElement GetElementById(this XDocument document, string id) + { + if (document == null) + { + throw new ArgumentNullException("document"); + } + + if (string.IsNullOrEmpty(id)) + { + throw new ArgumentNullException("id"); + } + + return + (from element in document.Descendants() + let idAttribute = element.Attribute("id") + where idAttribute != null && idAttribute.Value == id + select element).SingleOrDefault(); + } + + #endregion + + #region XElement extensions + + public static string GetId(this XElement element) + { + return element.GetAttributeValue("id", ""); + } + + public static void SetId(this XElement element, string id) + { + element.SetAttributeValue("id", id); + } + + public static string GetClass(this XElement element) + { + return element.GetAttributeValue("class", ""); + } + + public static void SetClass(this XElement element, string @class) + { + element.SetAttributeValue("class", @class); + } + + public static string GetStyle(this XElement element) + { + return element.GetAttributeValue("style", ""); + } + + public static void SetStyle(this XElement element, string style) + { + element.SetAttributeValue("style", style); + } + + public static string GetAttributeValue(this XElement element, string attributeName, string defaultValue) + { + if (element == null) + { + throw new ArgumentNullException("element"); + } + + if (string.IsNullOrEmpty(attributeName)) + { + throw new ArgumentNullException("attributeName"); + } + + var attribute = element.Attribute(attributeName); + + return attribute != null + ? (attribute.Value ?? defaultValue) + : defaultValue; + } + + public static void SetAttributeValue(this XElement element, string attributeName, string value) + { + if (element == null) + { + throw new ArgumentNullException("element"); + } + + if (string.IsNullOrEmpty(attributeName)) + { + throw new ArgumentNullException("attributeName"); + } + + if (value == null) + { + var attribute = element.Attribute(attributeName); + + if (attribute != null) + { + attribute.Remove(); + } + } + else + { + element.SetAttributeValue(attributeName, value); + } + } + + public static string GetAttributesString(this XElement element, string separator) + { + if (element == null) + { + throw new ArgumentNullException("element"); + } + + if (separator == null) + { + throw new ArgumentNullException("separator"); + } + + var resultSb = new StringBuilder(); + bool isFirst = true; + + element.Attributes().Aggregate( + resultSb, + (sb, attribute) => + { + string attributeValue = attribute.Value; + + if (string.IsNullOrEmpty(attributeValue)) + { + return sb; + } + + if (!isFirst) + { + resultSb.Append(separator); + } + + isFirst = false; + + sb.Append(attribute.Value); + + return sb; + }); + + return resultSb.ToString(); + } + + public static string GetInnerHtml(this XContainer container) + { + if (container == null) + { + throw new ArgumentNullException("container"); + } + + var resultSb = new StringBuilder(); + + foreach (var childNode in container.Nodes()) + { + resultSb.Append(childNode.ToString(SaveOptions.DisableFormatting)); + } + + return resultSb.ToString(); + } + + public static void SetInnerHtml(this XElement element, string html) + { + if (element == null) + { + throw new ArgumentNullException("element"); + } + + if (html == null) + { + throw new ArgumentNullException("html"); + } + + element.RemoveAll(); + + var tmpElement = new SgmlDomBuilder().BuildDocument(html); + + if (tmpElement.Root == null) + { + return; + } + + foreach (var node in tmpElement.Root.Nodes()) + { + element.Add(node); + } + } + + #endregion + + #region XContainer extensions + + public static IEnumerable GetElementsByTagName(this XContainer container, string tagName) + { + if (container == null) + { + throw new ArgumentNullException("container"); + } + + if (string.IsNullOrEmpty(tagName)) + { + throw new ArgumentNullException("tagName"); + } + + return container.Descendants() + .Where(e => tagName.Equals(e.Name.LocalName, StringComparison.OrdinalIgnoreCase)); + } + + public static IEnumerable GetChildrenByTagName(this XContainer container, string tagName) + { + if (container == null) + { + throw new ArgumentNullException("container"); + } + + if (string.IsNullOrEmpty(tagName)) + { + throw new ArgumentNullException("tagName"); + } + + return container.Elements() + .Where(e => e.Name != null && tagName.Equals(e.Name.LocalName, StringComparison.OrdinalIgnoreCase)); + } + + #endregion + } +} diff --git a/PortablePorts/NReadability/DomSerializationParams.cs b/PortablePorts/NReadability/DomSerializationParams.cs new file mode 100644 index 0000000..6831911 --- /dev/null +++ b/PortablePorts/NReadability/DomSerializationParams.cs @@ -0,0 +1,56 @@ + +namespace ReadSharp.Ports.NReadability +{ + public class DomSerializationParams + { + #region Factory methods + + /// + /// Creates an instance of DomSerializationParams with parameters set to their defaults. + /// + public static DomSerializationParams CreateDefault() + { + return new DomSerializationParams(); + } + + #endregion + + #region Properties + + /// + /// Determines whether the output will be formatted. + /// + public bool PrettyPrint { get; set; } + + /// + /// Determines whether DOCTYPE will be included at the beginning of the output. + /// + public bool DontIncludeContentTypeMetaElement { get; set; } + + /// + /// Determines whether mobile-specific elements (such as eg. meta HandheldFriendly) will be added/replaced in the output. + /// + public bool DontIncludeMobileSpecificMetaElements { get; set; } + + /// + /// Determines whether a meta tag with a content-type specification will be added/replaced in the output. + /// + public bool DontIncludeDocTypeMetaElement { get; set; } + + /// + /// Determines whether a meta tag with a generator specification will be added/replaced in the output. + /// + public bool DontIncludeGeneratorMetaElement { get; set; } + + /// + /// Render complete Website or only the Body + /// + public bool BodyOnly { get; set; } + + /// + /// Remove headline of website + /// + public bool NoHeadline { get; set; } + #endregion + } +} diff --git a/PortablePorts/NReadability/ElementsTraverser.cs b/PortablePorts/NReadability/ElementsTraverser.cs new file mode 100644 index 0000000..6bbbcc7 --- /dev/null +++ b/PortablePorts/NReadability/ElementsTraverser.cs @@ -0,0 +1,67 @@ +/* + * NReadability + * http://code.google.com/p/nreadability/ + * + * Copyright 2010 Marek Stój + * http://immortal.pl/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System; +using System.Xml.Linq; + +namespace ReadSharp.Ports.NReadability +{ + public class ElementsTraverser + { + private readonly Action _elementVisitor; + + #region Constructor(s) + + public ElementsTraverser(Action elementVisitor) + { + if (elementVisitor == null) + { + throw new ArgumentNullException("elementVisitor"); + } + + _elementVisitor = elementVisitor; + } + + #endregion + + #region Public methods + + public void Traverse(XElement element) + { + _elementVisitor(element); + + var childNode = element.FirstNode; + + while (childNode != null) + { + var nextChildNode = childNode.NextNode; + + if (childNode is XElement) + { + Traverse((XElement)childNode); + } + + childNode = nextChildNode; + } + } + + #endregion + } +} diff --git a/PortablePorts/NReadability/EncodedStringWriter.cs b/PortablePorts/NReadability/EncodedStringWriter.cs new file mode 100644 index 0000000..da48354 --- /dev/null +++ b/PortablePorts/NReadability/EncodedStringWriter.cs @@ -0,0 +1,62 @@ +/* + * NReadability + * http://code.google.com/p/nreadability/ + * + * Copyright 2010 Marek Stój + * http://immortal.pl/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System; +using System.IO; +using System.Text; + +namespace ReadSharp.Ports.NReadability +{ + internal class EncodedStringWriter : StringWriter + { + private static readonly Encoding _DefaultEncoding = Encoding.UTF8; + + private readonly Encoding _encoding; + + #region Constructor(s) + + public EncodedStringWriter(StringBuilder sb, Encoding encoding) + : base(sb) + { + if (encoding == null) + { + throw new ArgumentNullException("encoding"); + } + + _encoding = encoding; + } + + public EncodedStringWriter(StringBuilder sb) + : this(sb, _DefaultEncoding) + { + } + + #endregion + + #region Properties + + public override Encoding Encoding + { + get { return _encoding; } + } + + #endregion + } +} diff --git a/PortablePorts/NReadability/EnumerableExtensions.cs b/PortablePorts/NReadability/EnumerableExtensions.cs new file mode 100644 index 0000000..dd79c86 --- /dev/null +++ b/PortablePorts/NReadability/EnumerableExtensions.cs @@ -0,0 +1,43 @@ +using System; +using System.Collections.Generic; +using System.Linq; + +namespace ReadSharp.Ports.NReadability +{ + public static class EnumerableExtensions + { + /// + /// Returns the only one element in the sequence or default(T) if either the sequence doesn't contain any elements or it contains more than one element. + /// + public static T SingleOrNone(this IEnumerable enumerable) + where T : class + { + // ReSharper disable PossibleMultipleEnumeration + + if (enumerable == null) + { + throw new ArgumentNullException("enumerable"); + } + + T firstElement = enumerable.FirstOrDefault(); + + if (firstElement == null) + { + // no elements + return null; + } + + T secondElement = enumerable.Skip(1).FirstOrDefault(); + + if (secondElement != null) + { + // more than one element + return null; + } + + return firstElement; + + // ReSharper restore PossibleMultipleEnumeration + } + } +} diff --git a/PortablePorts/NReadability/Enums.cs b/PortablePorts/NReadability/Enums.cs new file mode 100644 index 0000000..76f36cd --- /dev/null +++ b/PortablePorts/NReadability/Enums.cs @@ -0,0 +1,110 @@ +/* + * NReadability + * http://code.google.com/p/nreadability/ + * + * Copyright 2010 Marek Stój + * http://immortal.pl/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +namespace ReadSharp.Ports.NReadability +{ + /// + /// Determines how the extracted article will be styled. + /// + public enum ReadingStyle + { + /// + /// Newspaper style. + /// + Newspaper, + + /// + /// Novel style. + /// + Novel, + + /// + /// Ebook style. + /// + Ebook, + + /// + /// Terminal style. + /// + Terminal, + } + + /// + /// Determines how wide the margin of the extracted article will be. + /// + public enum ReadingMargin + { + /// + /// Extra-narrow margin. + /// + XNarrow, + + /// + /// Narrow margin. + /// + Narrow, + + /// + /// Medium margin. + /// + Medium, + + /// + /// Wide margin. + /// + Wide, + + /// + /// Extra-wide margin. + /// + XWide, + } + + /// + /// Determines how large the font of the extracted article will be. + /// + public enum ReadingSize + { + /// + /// Extra-small font. + /// + XSmall, + + /// + /// Small font. + /// + Small, + + /// + /// Medium font. + /// + Medium, + + /// + /// Large font. + /// + Large, + + /// + /// Extra-large font. + /// + XLarge, + } +} diff --git a/PortablePorts/NReadability/HtmlUtils.cs b/PortablePorts/NReadability/HtmlUtils.cs new file mode 100644 index 0000000..427dc41 --- /dev/null +++ b/PortablePorts/NReadability/HtmlUtils.cs @@ -0,0 +1,40 @@ +using System; + +namespace ReadSharp.Ports.NReadability +{ + public static class HtmlUtils + { + public static string RemoveScriptTags(string htmlContent) + { + if (htmlContent == null) + { + throw new ArgumentNullException("htmlContent"); + } + + if (htmlContent.Length == 0) + { + return ""; + } + + int indexOfScriptTagStart = htmlContent.IndexOf("", indexOfScriptTagStart, StringComparison.OrdinalIgnoreCase); + + if (indexOfScriptTagEnd == -1) + { + return htmlContent.Substring(0, indexOfScriptTagStart); + } + + string strippedHtmlContent = + htmlContent.Substring(0, indexOfScriptTagStart) + + htmlContent.Substring(indexOfScriptTagEnd + "".Length); + + return RemoveScriptTags(strippedHtmlContent); + } + } +} diff --git a/PortablePorts/NReadability/InternalErrorException.cs b/PortablePorts/NReadability/InternalErrorException.cs new file mode 100644 index 0000000..8feae6d --- /dev/null +++ b/PortablePorts/NReadability/InternalErrorException.cs @@ -0,0 +1,62 @@ +/* + * NReadability + * http://code.google.com/p/nreadability/ + * + * Copyright 2010 Marek Stój + * http://immortal.pl/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System; +using System.Runtime.Serialization; + +namespace ReadSharp.Ports.NReadability +{ + /// + /// An exception that is thrown when an internal error occurrs in the application. + /// Internal error in the application means that there is a bug in the application. + /// + public class InternalErrorException : Exception + { + #region Constructor(s) + + /// + /// Initializes a new instance of the InternalErrorException class with a specified error message and a reference to the inner exception that is the cause of this exception. + /// + /// The error message that explains the reason for the exception. + /// The exception that is the cause of the current exception, or a null reference (Nothing in Visual Basic) if no inner exception is specified. + public InternalErrorException(string message, Exception innerException) + : base(message, innerException) + { + } + + /// + /// Initializes a new instance of the InternalErrorException class with a specified error message. + /// + /// The message that describes the error. + public InternalErrorException(string message) + : base(message) + { + } + + /// + /// Initializes a new instance of the InternalErrorException class. + /// + public InternalErrorException() + { + } + + #endregion + } +} diff --git a/PortablePorts/NReadability/NReadability.csproj b/PortablePorts/NReadability/NReadability.csproj new file mode 100644 index 0000000..026ac77 --- /dev/null +++ b/PortablePorts/NReadability/NReadability.csproj @@ -0,0 +1,76 @@ + + + + + 10.0 + Debug + AnyCPU + {14C3EE6A-54A4-4A37-8B56-D52A3802F1C2} + Library + Properties + ReadSharp.Ports.NReadability + ReadSharp.Ports.NReadability + v4.0 + Profile96 + 512 + {786C830F-07A1-408B-BD7F-6EE04809D6DB};{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC} + + + true + full + false + bin\Debug\ + DEBUG;TRACE + prompt + 4 + + + pdbonly + true + bin\Release\ + TRACE + prompt + 4 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + {9112414c-e2d1-43ba-a298-a89f77d94332} + SgmlReader + + + + + \ No newline at end of file diff --git a/PortablePorts/NReadability/NReadabilityTranscoder.cs b/PortablePorts/NReadability/NReadabilityTranscoder.cs new file mode 100644 index 0000000..1328dd9 --- /dev/null +++ b/PortablePorts/NReadability/NReadabilityTranscoder.cs @@ -0,0 +1,1840 @@ +/* + * NReadability + * http://code.google.com/p/nreadability/ + * + * Copyright 2010 Marek Stój + * http://immortal.pl/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Reflection; +using System.Text; +using System.Text.RegularExpressions; +using System.Xml; +using System.Xml.Linq; + +namespace ReadSharp.Ports.NReadability +{ + /// + /// A class that extracts main content from a HTML page. + /// + public class NReadabilityTranscoder + { + #region Nested types + + /// + /// Used in FindNextPageLink + /// + private class LinkData + { + public float Score; + public string LinkText; + public string LinkHref; + } + + #endregion + + private static readonly string _ReadabilityStylesheetResourceName = typeof(NReadabilityTranscoder).Namespace + ".Resources.readability.css"; + + #region Algorithm constants + + /// + /// Default styling of the extracted article. + /// + public const ReadingStyle DefaultReadingStyle = ReadingStyle.Newspaper; + + /// + /// Default margin of the extracted article. + /// + public const ReadingMargin DefaultReadingMargin = ReadingMargin.Wide; + + /// + /// Default size of the font used for the extracted article. + /// + public const ReadingSize DefaultReadingSize = ReadingSize.Medium; + + internal const string OverlayDivId = "readOverlay"; + internal const string InnerDivId = "readInner"; + internal const string ContentDivId = "readability-content"; + internal const string ReadabilityStyledCssClass = "readability-styled"; + + private const int _MinParagraphLength = 25; + private const int _MinInnerTextLength = 25; + private const int _ParagraphSegmentLength = 100; + private const int _MaxPointsForSegmentsCount = 3; + private const int _MinSiblingParagraphLength = 80; + private const int _MinCommaSegments = 10; + private const int _LisCountTreshold = 100; + private const int _MaxImagesInShortSegmentsCount = 2; + private const int _MinInnerTextLengthInElementsWithEmbed = 75; + private const int _ClassWeightTreshold = 25; + private const int _MaxEmbedsCount = 1; + private const int _MaxArticleTitleLength = 150; + private const int _MinArticleTitleLength = 15; + private const int _MinArticleTitleWordsCount1 = 3; + private const int _MinArticleTitleWordsCount2 = 4; + + private const float _SiblingScoreTresholdCoefficient = 0.2f; + private const float _MaxSiblingScoreTreshold = 10.0f; + private const float _MaxSiblingParagraphLinksDensity = 0.25f; + private const float _MaxHeaderLinksDensity = 0.33f; + private const float _MaxDensityForElementsWithSmallerClassWeight = 0.2f; + private const float _MaxDensityForElementsWithGreaterClassWeight = 0.5f; + + #endregion + + #region Algorithm regular expressions + + private static readonly Regex _UnlikelyCandidatesRegex = new Regex("combx|comment|community|disqus|extra|foot|header|menu|remark|rss|shoutbox|sidebar|side|sponsor|ad-break|agegate|pagination|pager|popup|tweet|twitter", RegexOptions.IgnoreCase); + private static readonly Regex _OkMaybeItsACandidateRegex = new Regex("and|article|body|column|main|shadow", RegexOptions.IgnoreCase); + private static readonly Regex _PositiveWeightRegex = new Regex("article|body|content|entry|hentry|main|page|pagination|post|text|blog|story", RegexOptions.IgnoreCase); + private static readonly Regex _NegativeWeightRegex = new Regex("combx|comment|com-|contact|foot|footer|footnote|masthead|media|meta|outbrain|promo|related|scroll|shoutbox|sidebar|side|sponsor|shopping|tags|tool|widget", RegexOptions.IgnoreCase); + private static readonly Regex _NegativeLinkParentRegex = new Regex("(stories|articles|news|documents|posts|notes|series|historie|artykuly|artykuły|wpisy|dokumenty|serie|geschichten|erzählungen|erzahlungen)", RegexOptions.IgnoreCase); + private static readonly Regex _Extraneous = new Regex("print|archive|comment|discuss|e[-]?mail|share|reply|all|login|sign|single|also", RegexOptions.IgnoreCase); + private static readonly Regex _DivToPElementsRegex = new Regex("<(a|blockquote|dl|div|img|ol|p|pre|table|ul)", RegexOptions.IgnoreCase); + private static readonly Regex _EndOfSentenceRegex = new Regex("\\.( |$)", RegexOptions.Multiline); + private static readonly Regex _BreakBeforeParagraphRegex = new Regex("]*>\\s*(\\s| ?)*){1,}"); + private static readonly Regex _VideoRegex = new Regex("http:\\/\\/(www\\.)?(youtube|vimeo)\\.com", RegexOptions.IgnoreCase); + private static readonly Regex _ReplaceDoubleBrsRegex = new Regex("(]*>[ \\n\\r\\t]*){2,}", RegexOptions.IgnoreCase); + private static readonly Regex _ReplaceFontsRegex = new Regex("<(\\/?)font[^>]*>", RegexOptions.IgnoreCase); + private static readonly Regex _ArticleTitleDashRegex1 = new Regex(" [\\|\\-] "); + private static readonly Regex _ArticleTitleDashRegex2 = new Regex("(.*)[\\|\\-] .*"); + private static readonly Regex _ArticleTitleDashRegex3 = new Regex("[^\\|\\-]*[\\|\\-](.*)"); + private static readonly Regex _ArticleTitleColonRegex1 = new Regex(".*:(.*)"); + private static readonly Regex _ArticleTitleColonRegex2 = new Regex("[^:]*[:](.*)"); + private static readonly Regex _NextLink = new Regex(@"(next|weiter|continue|dalej|następna|nastepna>([^\|]|$)|�([^\|]|$))", RegexOptions.IgnoreCase); + private static readonly Regex _NextStoryLink = new Regex("(story|article|news|document|post|note|series|historia|artykul|artykuł|wpis|dokument|seria|geschichte|erzählung|erzahlung|artikel|serie)", RegexOptions.IgnoreCase); + private static readonly Regex _PrevLink = new Regex("(prev|earl|[^b]old|new|wstecz|poprzednia|<|�)", RegexOptions.IgnoreCase); + private static readonly Regex _PageRegex = new Regex("pag(e|ing|inat)|([^a-z]|^)pag([^a-z]|$)", RegexOptions.IgnoreCase); + private static readonly Regex _LikelyParagraphDivRegex = new Regex("text|para|parbase", RegexOptions.IgnoreCase); + + #endregion + + #region Other regular expressions + + private static readonly Regex _MailtoHrefRegex = new Regex("^\\s*mailto\\s*:", RegexOptions.IgnoreCase); + private static readonly Regex _TitleWhitespacesCleanUpRegex = new Regex("\\s+"); + + private static readonly Dictionary _articleContentElementHints = + new Dictionary + { + { new Regex("^https?://(www|mobile)\\.theverge.com", RegexOptions.IgnoreCase), "article" }, + }; + + #endregion + + #region Algorithm parameters + + private bool _dontStripUnlikelys; + private readonly bool _dontNormalizeSpacesInTextContent; + private readonly bool _dontWeightClasses; + private readonly ReadingStyle _readingStyle; + private readonly ReadingSize _readingSize; + private readonly ReadingMargin _readingMargin; + + #endregion + + #region Helper instance fields + + private readonly SgmlDomBuilder _sgmlDomBuilder; + private readonly SgmlDomSerializer _sgmlDomSerializer; + private readonly Dictionary _elementsScores; + + private Func _imageSourceTranformer; + private Func _anchorHrefTransformer; + + #endregion + + #region Constructor(s) + + /// + /// Initializes a new instance of NReadabilityTranscoder. Allows setting all options. + /// + /// Determines whether elements that are unlikely to be a part of main content will be removed. + /// Determines whether spaces in InnerText properties of elements will be normalized automatically (eg. whether double spaces will be replaced with single spaces). + /// Determines whether 'weight-class' algorithm will be used when cleaning content. + /// Styling for the extracted article. + /// Margin for the extracted article. + /// Font size for the extracted article. + public NReadabilityTranscoder( + bool dontStripUnlikelys, + bool dontNormalizeSpacesInTextContent, + bool dontWeightClasses, + ReadingStyle readingStyle, + ReadingMargin readingMargin, + ReadingSize readingSize) + { + _dontStripUnlikelys = dontStripUnlikelys; + _dontNormalizeSpacesInTextContent = dontNormalizeSpacesInTextContent; + _dontWeightClasses = dontWeightClasses; + _readingStyle = readingStyle; + _readingMargin = readingMargin; + _readingSize = readingSize; + + _sgmlDomBuilder = new SgmlDomBuilder(); + _sgmlDomSerializer = new SgmlDomSerializer(); + _elementsScores = new Dictionary(); + } + + /// + /// Initializes a new instance of NReadabilityTranscoder. Allows setting reading options. + /// + /// Styling for the extracted article. + /// Margin for the extracted article. + /// Font size for the extracted article. + public NReadabilityTranscoder(ReadingStyle readingStyle, ReadingMargin readingMargin, ReadingSize readingSize) + : this(false, false, false, readingStyle, readingMargin, readingSize) + { + } + + /// + /// Initializes a new instance of NReadabilityTranscoder. + /// + public NReadabilityTranscoder() + : this(DefaultReadingStyle, DefaultReadingMargin, DefaultReadingSize) + { + } + + #endregion + + #region Public methods + + /// + /// Extracts article content from an HTML page. + /// + /// An object containing input parameters, i.a. html content to be processed. + /// An object containing transcoding result, i.a. extracted content and title. + public TranscodingResult Transcode(TranscodingInput transcodingInput) + { + if (transcodingInput == null) + { + throw new ArgumentNullException("transcodingInput"); + } + + bool contentExtracted; + string extractedTitle; + string nextPageUrl; + + XDocument transcodedXmlDocument = + TranscodeToXml( + transcodingInput.HtmlContent, + transcodingInput.Url, + out contentExtracted, + out extractedTitle, + out nextPageUrl); + + IEnumerable images = transcodedXmlDocument.GetElementsByTagName("img"); + + string transcodedContent = + _sgmlDomSerializer.SerializeDocument( + transcodedXmlDocument, + transcodingInput.DomSerializationParams); + + bool titleExtracted = !string.IsNullOrEmpty(extractedTitle); + + return + new TranscodingResult(contentExtracted, titleExtracted) + { + ExtractedContent = transcodedContent, + ExtractedTitle = extractedTitle, + NextPageUrl = nextPageUrl, + Images = images + }; + } + + /// + /// Extracts main article content from a HTML page. + /// + /// HTML markup to process. + /// Url from which the content was downloaded. Used to resolve relative urls. Can be null. + /// Contains parameters that modify the behaviour of the output serialization. + /// Determines whether the content has been extracted (if the article is not empty). + /// If the content contains a link to a subsequent page, it is returned here. + /// HTML markup containing extracted article content. + [Obsolete("Use TranscodingResult Transcode(TranscodingInput) method.")] + public string Transcode(string htmlContent, string url, DomSerializationParams domSerializationParams, out bool mainContentExtracted, out string nextPageUrl) + { + string extractedTitle; + + XDocument document = + TranscodeToXml( + htmlContent, + url, + out mainContentExtracted, + out extractedTitle, + out nextPageUrl); + + return _sgmlDomSerializer.SerializeDocument(document, domSerializationParams); + } + + /// + /// Extracts main article content from a HTML page. + /// + /// HTML markup to process. + /// Url from which the content was downloaded. Used to resolve relative urls. Can be null. + /// Determines whether the content has been extracted (if the article is not empty). + /// If the content contains a link to a subsequent page, it is returned here. + /// HTML markup containing extracted article content. + [Obsolete("Use TranscodingResult Transcode(TranscodingInput) method.")] + public string Transcode(string htmlContent, string url, out bool mainContentExtracted, out string nextPageUrl) + { + return Transcode(htmlContent, url, DomSerializationParams.CreateDefault(), out mainContentExtracted, out nextPageUrl); + } + + /// + /// Extracts main article content from a HTML page. + /// + /// HTML markup to process. + /// Url from which the content was downloaded. Used to resolve relative urls. Can be null. + /// Determines whether the content has been extracted (if the article is not empty). + /// HTML markup containing extracted article content. + [Obsolete("Use TranscodingResult Transcode(TranscodingInput) method.")] + public string Transcode(string htmlContent, string url, out bool mainContentExtracted) + { + string nextPageUrl; + + return Transcode(htmlContent, url, DomSerializationParams.CreateDefault(), out mainContentExtracted, out nextPageUrl); + } + + /// + /// Extracts main article content from a HTML page. + /// + /// HTML markup to process. + /// Determines whether the content has been extracted (if the article is not empty). + /// HTML markup containing extracted article content. + [Obsolete("Use TranscodingResult Transcode(TranscodingInput) method.")] + public string Transcode(string htmlContent, out bool mainContentExtracted) + { + string nextPageUrl; + + return Transcode(htmlContent, null, out mainContentExtracted, out nextPageUrl); + } + + #endregion + + #region Readability algorithm + + /// + /// Extracts main article content from a HTML page. + /// + /// HTML markup to process. + /// Url from which the content was downloaded. Used to resolve relative urls. Can be null. + /// Determines whether the content has been extracted (if the article is not empty). + /// Will contain article title (if we were able to extract it). + /// If the content contains a link to a subsequent page, it is returned here. + /// An XDocument containing extracted article content. + internal XDocument TranscodeToXml(string htmlContent, string url, out bool mainContentExtracted, out string extractedTitle, out string nextPageUrl) + { + if (string.IsNullOrEmpty(htmlContent)) + { + throw new ArgumentNullException("htmlContent"); + } + + XDocument document = _sgmlDomBuilder.BuildDocument(htmlContent); + + PrepareDocument(document); + + if (!string.IsNullOrEmpty(url)) + { + ResolveElementsUrls(document, "img", "src", url, _imageSourceTranformer); + ResolveElementsUrls(document, "a", "href", url, _anchorHrefTransformer); + } + + nextPageUrl = null; + + if (!string.IsNullOrEmpty(url)) + { + nextPageUrl = FindNextPageLink(document.GetBody(), url); + } + + XElement articleTitleElement = ExtractArticleTitle(document); + XElement articleContentElement = ExtractArticleContent(document, url); + + GlueDocument(document, articleTitleElement, articleContentElement); + + // fallback behaviour - rerun one more time with _dontStripUnlikelys if we have little content + if (!_dontStripUnlikelys && GetInnerText(articleContentElement).Length < 250) + { + try + { + _dontStripUnlikelys = true; + + return TranscodeToXml(htmlContent, url, out mainContentExtracted, out extractedTitle, out nextPageUrl); + } + finally + { + _dontStripUnlikelys = false; + } + } + + // TODO: implement another fallback behaviour - rerun one more time with _dontWeightClasses + + mainContentExtracted = !articleContentElement.IsEmpty; + extractedTitle = ExtractTitle(document); + + return document; + } + + /// + /// Looks for any paging links that may occur within the document + /// + /// Content body + /// Url of document + internal string FindNextPageLink(XElement body, string url) + { + Dictionary possiblePagesByLink = new Dictionary(); + IEnumerable allLinks = body.GetElementsByTagName("a"); + string articleBaseUrl = FindBaseUrl(url); + + /* Loop through all links, looking for hints that they may be next-page links. + * Things like having "page" in their textContent, className or id, or being a child + * of a node with a page-y className or id. + * After we do that, assign each page a score. + */ + foreach (XElement linkElement in allLinks) + { + string linkHref = (string)linkElement.Attribute("href"); + + if (string.IsNullOrEmpty(linkHref) + || _MailtoHrefRegex.IsMatch(linkHref)) + { + continue; + } + + linkHref = Regex.Replace(linkHref, "#.*$", ""); + linkHref = Regex.Replace(linkHref, "/$", ""); + + /* If we've already seen this page, then ignore it. */ + // This leaves out an already-checked page check, because + // the web transcoder is seperate from the original transcoder + if (linkHref == "" || linkHref == articleBaseUrl || linkHref == url) + { + continue; + } + + /* If it's on a different domain, skip it. */ + Uri linkHrefUri; + + if (Uri.TryCreate(linkHref, UriKind.Absolute, out linkHrefUri) && linkHrefUri.Host != new Uri(articleBaseUrl).Host) + { + continue; + } + + string linkText = GetInnerText(linkElement); + + /* If the linktext looks like it's not the next page, then skip it */ + if (_Extraneous.IsMatch(linkText)) + { + continue; + } + + /* If the leftovers of the URL after removing the base URL don't contain any digits, it's certainly not a next page link. */ + string linkHrefLeftover = linkHref.Replace(articleBaseUrl, ""); + + if (!Regex.IsMatch(linkHrefLeftover, @"\d")) + { + continue; + } + + if (!possiblePagesByLink.Keys.Contains(linkHref)) + { + possiblePagesByLink[linkHref] = new LinkData { Score = 0, LinkHref = linkHref, LinkText = linkText }; + } + else + { + possiblePagesByLink[linkHref].LinkText += " | " + linkText; + } + + LinkData linkObj = possiblePagesByLink[linkHref]; + + /* + * If the articleBaseUrl isn't part of this URL, penalize this link. It could still be the link, but the odds are lower. + * Example: http://www.actionscript.org/resources/articles/745/1/JavaScript-and-VBScript-Injection-in-ActionScript-3/Page1.html + */ + if (linkHref.IndexOf(articleBaseUrl, StringComparison.OrdinalIgnoreCase) == -1) + { + linkObj.Score -= 25; + } + + string linkData = linkText + " " + linkElement.GetClass() + " " + linkElement.GetId(); + + if (_NextLink.IsMatch(linkData) + && !_NextStoryLink.IsMatch(linkData)) + { + linkObj.Score += 50; + } + + if (_PageRegex.IsMatch(linkData)) + { + linkObj.Score += 25; + } + + /* If we already matched on "next", last is probably fine. If we didn't, then it's bad. Penalize. */ + /* -65 is enough to negate any bonuses gotten from a > or � in the text */ + if (Regex.IsMatch(linkData, "(first|last)", RegexOptions.IgnoreCase) + && !_NextLink.IsMatch(linkObj.LinkText)) + { + linkObj.Score -= 65; + } + + if (_NegativeWeightRegex.IsMatch(linkData) || _Extraneous.IsMatch(linkData)) + { + linkObj.Score -= 50; + } + + if (_PrevLink.IsMatch(linkData)) + { + linkObj.Score -= 200; + } + + /* If any ancestor node contains page or paging or paginat */ + XElement parentNode = linkElement.Parent; + bool positiveNodeMatch = false; + bool negativeNodeMatch = false; + + while (parentNode != null) + { + string parentNodeClassAndId = parentNode.GetClass() + " " + parentNode.GetId(); + + if (!positiveNodeMatch && (_PageRegex.IsMatch(parentNodeClassAndId) || _NextLink.IsMatch(parentNodeClassAndId))) + { + positiveNodeMatch = true; + linkObj.Score += 25; + } + + if (!negativeNodeMatch && (_NegativeWeightRegex.IsMatch(parentNodeClassAndId) || _NegativeLinkParentRegex.IsMatch(parentNodeClassAndId))) + { + if (!_PositiveWeightRegex.IsMatch(parentNodeClassAndId)) + { + linkObj.Score -= 25; + negativeNodeMatch = true; + } + } + + parentNode = parentNode.Parent; + } + + /* If any descendant node contains 'next indicator' or 'prev indicator' - adjust the score */ + bool positiveDescendantMatch = false; + bool negativeDescendantMatch = false; + + foreach (XElement descendantElement in linkElement.Descendants()) + { + string descendantData = GetInnerText(descendantElement) + " " + descendantElement.GetClass() + " " + descendantElement.GetId() + " " + descendantElement.GetAttributeValue("alt", ""); + + if (!positiveDescendantMatch && _NextLink.IsMatch(descendantData)) + { + linkObj.Score += 12.5f; + positiveDescendantMatch = true; + } + + if (!negativeDescendantMatch && _PrevLink.IsMatch(descendantData)) + { + linkObj.Score -= 100; + negativeDescendantMatch = true; + } + } + + /* + * If the URL looks like it has paging in it, add to the score. + * Things like /page/2/, /pagenum/2, ?p=3, ?page=11, ?pagination=34 + */ + if (Regex.IsMatch(linkHref, @"p(a|g|ag)?(e|ing|ination)?(=|\/)[0-9]{1,2}", RegexOptions.IgnoreCase) + || Regex.IsMatch(linkHref, @"(page|paging)", RegexOptions.IgnoreCase) + || Regex.IsMatch(linkHref, @"section", RegexOptions.IgnoreCase)) + { + linkObj.Score += 25; + } + + /* If the URL contains negative values, give a slight decrease. */ + if (_Extraneous.IsMatch(linkHref)) + { + linkObj.Score -= 15; + } + + /* + * If the link text can be parsed as a number, give it a minor bonus, with a slight + * bias towards lower numbered pages. This is so that pages that might not have 'next' + * in their text can still get scored, and sorted properly by score. + */ + int linkTextAsNumber; + bool isInt = int.TryParse(linkText, out linkTextAsNumber); + + if (isInt) + { + /* Punish 1 since we're either already there, or it's probably before what we want anyways. */ + if (linkTextAsNumber == 1) + { + linkObj.Score -= 10; + } + else + { + linkObj.Score += Math.Max(0, 10 - linkTextAsNumber); + } + } + } + + /* + * Loop through all of our possible pages from above and find our top candidate for the next page URL. + * Require at least a score of 50, which is a relatively high confidence that this page is the next link. + */ + LinkData topPage = null; + + foreach (string page in possiblePagesByLink.Keys) + { + if (possiblePagesByLink[page].Score >= 50 && (topPage == null || topPage.Score < possiblePagesByLink[page].Score)) + { + topPage = possiblePagesByLink[page]; + } + } + + if (topPage != null) + { + string nextHref = Regex.Replace(topPage.LinkHref, @"\/$", ""); + var nextHrefUri = new Uri(new Uri(articleBaseUrl), nextHref); + + return nextHrefUri.OriginalString; + } + + return null; + } + + /// + /// Find a cleaned up version of the current URL, to use for comparing links for possible next-pageyness. + /// + internal string FindBaseUrl(string url) + { + Uri urlUri; + + if (!Uri.TryCreate(url, UriKind.Absolute, out urlUri)) + { + return url; + } + + string protocol = urlUri.Scheme; + string hostname = urlUri.Host; + string noUrlParams = urlUri.AbsolutePath + "/"; + List urlSlashes = noUrlParams.Split('/').Reverse().ToList(); + var cleanedSegments = new List(); + int slashLen = urlSlashes.Count(); + + for (int i = 0; i < slashLen; i++) + { + string segment = urlSlashes[i]; + + /* Split off and save anything that looks like a file type. */ + if (segment.IndexOf('.') != -1) + { + string possibleType = segment.Split('.')[1]; + + /* If the type isn't alpha-only, it's probably not actually a file extension. */ + if (!Regex.IsMatch(possibleType, "[^a-zA-Z]")) + { + segment = segment.Split('.')[0]; + } + } + + /* + * EW-CMS specific segment replacement. Ugly. + * Example: http://www.ew.com/ew/article/0,,20313460_20369436,00.html + */ + if (segment.IndexOf(",00") != -1) + { + segment = segment.Replace(",00", ""); + } + + /* If our first or second segment has anything looking like a page number, remove it. */ + var pageNumRegex = new Regex("((_|-)?p[a-z]*|(_|-))[0-9]{1,2}$", RegexOptions.IgnoreCase); + + if (pageNumRegex.IsMatch(segment) && ((i == 1) || (i == 0))) + { + segment = pageNumRegex.Replace(segment, ""); + } + + /* If this is purely a number, and it's the first or second segment, it's probably a page number. Remove it. */ + bool del = (i < 2 && Regex.IsMatch(segment, @"^[\d]{1,2}$")); + + /* If this is the first segment and it's just "index," remove it. */ + if (i == 0 && segment.ToLower() == "index") + { + del = true; + } + + /* If tour first or second segment is smaller than 3 characters, and the first segment was purely alphas, remove it. */ + // TODO: Check these "purely alpha" regexes. They don't seem right. + if (i < 2 && segment.Length < 3 && !Regex.IsMatch(urlSlashes[0], "[a-z]", RegexOptions.IgnoreCase)) + { + del = true; + } + + /* If it's not marked for deletion, push it to cleanedSegments */ + if (!del) + { + cleanedSegments.Add(segment); + } + } + + /* This is our final, cleaned, base article URL. */ + cleanedSegments.Reverse(); + + return string.Format("{0}://{1}{2}", protocol, hostname, String.Join("/", cleanedSegments.ToArray())); + } + + internal void PrepareDocument(XDocument document) + { + /* Remove all HTML comments (including conditional comments). */ + document + .DescendantNodes() + .Where(node => node.NodeType == XmlNodeType.Comment) + .Remove(); + + /* In some cases a body element can't be found (if the HTML is totally hosed for example), + * so we create a new body element and append it to the document. */ + XElement documentBody = GetOrCreateBody(document); + XElement rootElement = document.Root; + + // TODO: handle HTML frames + + var elementsToRemove = new List(); + + /* Remove all scripts that are not readability. */ + elementsToRemove.Clear(); + + rootElement.GetElementsByTagName("script") + .ForEach(scriptElement => + { + string scriptSrc = scriptElement.GetAttributeValue("src", null); + + if (string.IsNullOrEmpty(scriptSrc) || scriptSrc.LastIndexOf("readability") == -1) + { + elementsToRemove.Add(scriptElement); + } + }); + + RemoveElements(elementsToRemove); + + /* Remove all noscript tags. */ + elementsToRemove.Clear(); + elementsToRemove.AddRange(rootElement.GetElementsByTagName("noscript")); + RemoveElements(elementsToRemove); + + /* Remove all external stylesheets. */ + elementsToRemove.Clear(); + elementsToRemove.AddRange( + rootElement.GetElementsByTagName("link") + .Where(element => element.GetAttributeValue("rel", "").Trim().ToLower() == "stylesheet" + && element.GetAttributeValue("href", "").LastIndexOf("readability") == -1)); + RemoveElements(elementsToRemove); + + /* Remove all style tags. */ + elementsToRemove.Clear(); + elementsToRemove.AddRange(rootElement.GetElementsByTagName("style")); + RemoveElements(elementsToRemove); + + /* Remove all nav tags. */ + elementsToRemove.Clear(); + elementsToRemove.AddRange(rootElement.GetElementsByTagName("nav")); + RemoveElements(elementsToRemove); + + /* Remove all anchors. */ + elementsToRemove.Clear(); + + IEnumerable anchorElements = + rootElement.GetElementsByTagName("a") + .Where(aElement => aElement.Attribute("name") != null && aElement.Attribute("href") == null); + + elementsToRemove.AddRange(anchorElements); + RemoveElements(elementsToRemove); + + /* Turn all double br's into p's and all font's into span's. */ + // TODO: optimize? + string bodyInnerHtml = documentBody.GetInnerHtml(); + + bodyInnerHtml = _ReplaceDoubleBrsRegex.Replace(bodyInnerHtml, "

"); + bodyInnerHtml = _ReplaceFontsRegex.Replace(bodyInnerHtml, "<$1span>"); + + documentBody.SetInnerHtml(bodyInnerHtml); + } + + internal XElement ExtractArticleTitle(XDocument document) + { + XElement documentBody = GetOrCreateBody(document); + string documentTitle = document.GetTitle() ?? ""; + string currentTitle = documentTitle; + + if (_ArticleTitleDashRegex1.IsMatch(currentTitle)) + { + currentTitle = _ArticleTitleDashRegex2.Replace(documentTitle, "$1"); + + if (currentTitle.Split(' ').Length < _MinArticleTitleWordsCount1) + { + currentTitle = _ArticleTitleDashRegex3.Replace(documentTitle, "$1"); + } + } + else if (currentTitle.IndexOf(": ") != -1) + { + currentTitle = _ArticleTitleColonRegex1.Replace(documentTitle, "$1"); + + if (currentTitle.Split(' ').Length < _MinArticleTitleWordsCount1) + { + currentTitle = _ArticleTitleColonRegex2.Replace(documentTitle, "$1"); + } + } + else if (currentTitle.Length > _MaxArticleTitleLength || currentTitle.Length < _MinArticleTitleLength) + { + List titleHeaders = documentBody.GetElementsByTagName("h1").ToList(); + + if (titleHeaders.Count == 0) + { + // if we don't have any level one headers let's give level two header a chance + titleHeaders = documentBody.GetElementsByTagName("h2").ToList(); + } + + if (titleHeaders.Count == 1) + { + currentTitle = GetInnerText(titleHeaders[0]); + } + } + + currentTitle = (currentTitle ?? "").Trim(); + + if (!string.IsNullOrEmpty(documentTitle) + && currentTitle.Split(' ').Length <= _MinArticleTitleWordsCount2) + { + currentTitle = documentTitle; + } + + if (string.IsNullOrEmpty(currentTitle)) + { + return null; + } + + var articleTitleElement = new XElement("h1"); + + articleTitleElement.SetInnerHtml(currentTitle); + + return articleTitleElement; + } + + internal XElement ExtractArticleContent(XDocument document, string url = null) + { + StripUnlikelyCandidates(document); + CollapseRedundantParagraphDivs(document); + + string articleContentElementHint = + !string.IsNullOrEmpty(url) + ? GetArticleContentElementHint(url) + : null; + + IEnumerable candidatesForArticleContent = + FindCandidatesForArticleContent( + document, + articleContentElementHint); + + XElement topCandidateElement = DetermineTopCandidateElement(document, candidatesForArticleContent); + XElement articleContentElement = CreateArticleContentElement(document, topCandidateElement); + + PrepareArticleContentElement(articleContentElement); + + return articleContentElement; + } + + internal void GlueDocument(XDocument document, XElement articleTitleElement, XElement articleContentElement) + { + XElement documentBody = GetOrCreateBody(document); + + /* Include readability.css stylesheet. */ + XElement headElement = document.GetElementsByTagName("head").FirstOrDefault(); + + if (headElement == null) + { + headElement = new XElement("head"); + documentBody.AddBeforeSelf(headElement); + } + + XElement styleElement = new XElement("style"); + + styleElement.SetAttributeValue("type", "text/css"); + + Stream readabilityStylesheetStream = Assembly.GetExecutingAssembly().GetManifestResourceStream(_ReadabilityStylesheetResourceName); + + if (readabilityStylesheetStream == null) + { + throw new InternalErrorException("Couldn't load the NReadability stylesheet embedded resource."); + } + + using (var sr = new StreamReader(readabilityStylesheetStream)) + { + styleElement.SetInnerHtml(sr.ReadToEnd()); + } + + headElement.Add(styleElement); + + /* Apply reading style to body. */ + string readingStyleClass = GetReadingStyleClass(_readingStyle); + + documentBody.SetClass(readingStyleClass); + documentBody.SetStyle("display: block;"); + + /* Create inner div. */ + var innerDiv = new XElement("div"); + + innerDiv.SetId(InnerDivId); + innerDiv.SetClass(GetReadingMarginClass(_readingMargin) + " " + GetReadingSizeClass(_readingSize)); + + if (articleTitleElement != null) + { + innerDiv.Add(articleTitleElement); + } + + if (articleContentElement != null) + { + innerDiv.Add(articleContentElement); + } + + /* Create overlay div. */ + var overlayDiv = new XElement("div"); + + overlayDiv.SetId(OverlayDivId); + overlayDiv.SetClass(readingStyleClass); + overlayDiv.Add(innerDiv); + + /* Clear the old HTML, insert the new content. */ + documentBody.RemoveAll(); + documentBody.Add(overlayDiv); + } + + internal void StripUnlikelyCandidates(XDocument document) + { + if (_dontStripUnlikelys) + { + return; + } + + XElement rootElement = document.Root; + + new ElementsTraverser( + element => + { + string elementName = GetElementName(element); + + /* Remove unlikely candidates. */ + string unlikelyMatchString = element.GetClass() + " " + element.GetId(); + + if (unlikelyMatchString.Length > 0 + && !"body".Equals(elementName, StringComparison.OrdinalIgnoreCase) + && !"a".Equals(elementName, StringComparison.OrdinalIgnoreCase) + && _UnlikelyCandidatesRegex.IsMatch(unlikelyMatchString) + && !_OkMaybeItsACandidateRegex.IsMatch(unlikelyMatchString)) + { + XElement parentElement = element.Parent; + + if (parentElement != null) + { + element.Remove(); + } + + // element has been removed - we can go to the next one + return; + } + + /* Turn all divs that don't have children block level elements into p's or replace text nodes within the div with p's. */ + if ("div".Equals(elementName, StringComparison.OrdinalIgnoreCase)) + { + if (!_DivToPElementsRegex.IsMatch(element.GetInnerHtml())) + { + // no block elements inside - change to p + SetElementName(element, "p"); + } + else + { + // replace text nodes with p's (experimental) + new ChildNodesTraverser( + childNode => + { + if (childNode.NodeType != XmlNodeType.Text + || GetInnerText(childNode).Length == 0) + { + return; + } + + XElement paraElement = new XElement("p"); + + // note that we're not using GetInnerText() here; instead we're getting raw InnerText to preserve whitespaces + paraElement.SetInnerHtml(((XText)childNode).Value); + + paraElement.SetClass(ReadabilityStyledCssClass); + paraElement.SetStyle("display: inline;"); + + childNode.ReplaceWith(paraElement); + } + ).Traverse(element); + } + } + }).Traverse(rootElement); + } + + internal void CollapseRedundantParagraphDivs(XDocument document) + { + XElement rootElement = document.Root; + + new ElementsTraverser( + element => + { + string elementName = GetElementName(element); + + if ("div".Equals(elementName, StringComparison.OrdinalIgnoreCase)) + { + XNode childNode = element.Nodes().SingleOrNone(); + + if (childNode != null) + { + XElement childElement = childNode as XElement; + + if (childElement != null && "p".Equals(GetElementName(childElement), StringComparison.OrdinalIgnoreCase)) + { + // we have a div with a single child element that is a paragraph - let's remove the div and attach the paragraph to the div's parent + XElement parentElement = element.Parent; + + if (parentElement != null) + { + element.AddBeforeSelf(childElement); + element.Remove(); + } + } + } + } + }).Traverse(rootElement); + } + + internal IEnumerable FindCandidatesForArticleContent(XDocument document, string articleContentElementHint = null) + { + if (!string.IsNullOrEmpty(articleContentElementHint)) + { + XElement articleContentElement = + TryFindArticleContentElement(document, articleContentElementHint); + + if (articleContentElement != null) + { + return new[] { articleContentElement }; + } + } + + IEnumerable paraElements = document.GetElementsByTagName("p"); + var candidateElements = new List(); + + _elementsScores.Clear(); + + foreach (XElement paraElement in paraElements) + { + string innerText = GetInnerText(paraElement); + + if (innerText.Length < _MinParagraphLength) + { + continue; + } + + XElement parentElement = paraElement.Parent; + XElement grandParentElement = parentElement != null ? parentElement.Parent : null; + int score = 1; // 1 point for having a paragraph + + // Add points for any comma-segments within this paragraph. + score += GetSegmentsCount(innerText, ','); + + // For every PARAGRAPH_SEGMENT_LENGTH characters in this paragraph, add another point. Up to MAX_POINTS_FOR_SEGMENTS_COUNT points. + score += Math.Min(innerText.Length / _ParagraphSegmentLength, _MaxPointsForSegmentsCount); + + // Add the score to the parent. + if (parentElement != null && !"html".Equals(parentElement.Name.LocalName, StringComparison.OrdinalIgnoreCase)) + { + candidateElements.Add(parentElement); + AddPointsToElementScore(parentElement, score); + } + + // Add half the score to the grandparent. + if (grandParentElement != null && !"html".Equals(grandParentElement.Name.LocalName, StringComparison.OrdinalIgnoreCase)) + { + candidateElements.Add(grandParentElement); + AddPointsToElementScore(grandParentElement, score / 2); + } + } + + return candidateElements; + } + + internal XElement DetermineTopCandidateElement(XDocument document, IEnumerable candidatesForArticleContent) + { + XElement topCandidateElement = null; + + foreach (XElement candidateElement in candidatesForArticleContent) + { + float candidateScore = GetElementScore(candidateElement); + + // Scale the final candidates score based on link density. Good content should have a + // relatively small link density (5% or less) and be mostly unaffected by this operation. + float newCandidateScore = (1.0f - GetLinksDensity(candidateElement)) * candidateScore; + + SetElementScore(candidateElement, newCandidateScore); + + if (topCandidateElement == null + || newCandidateScore > GetElementScore(topCandidateElement)) + { + topCandidateElement = candidateElement; + } + } + + if (topCandidateElement == null + || "body".Equals(topCandidateElement.Name.LocalName, StringComparison.OrdinalIgnoreCase)) + { + topCandidateElement = new XElement("div"); + + XElement documentBody = GetOrCreateBody(document); + + topCandidateElement.Add(documentBody.Nodes()); + } + + return topCandidateElement; + } + + internal XElement CreateArticleContentElement(XDocument document, XElement topCandidateElement) + { + /* Now that we have the top candidate, look through its siblings for content that might also be related. + * Things like preambles, content split by ads that we removed, etc. */ + + var articleContentElement = new XElement("div"); + + articleContentElement.SetId(ContentDivId); + + XElement parentElement = topCandidateElement.Parent; + + if (parentElement == null) + { + // if the top candidate element has no parent, it means that it's an element created by us and detached from the document, + // so we don't analyze its siblings and just attach it to the article content + articleContentElement.Add(topCandidateElement); + + return articleContentElement; + } + + IEnumerable siblingElements = parentElement.Elements(); + + float topCandidateElementScore = GetElementScore(topCandidateElement); + float siblingScoreThreshold = + Math.Max( + _MaxSiblingScoreTreshold, + _SiblingScoreTresholdCoefficient * topCandidateElementScore); + + string topCandidateClass = topCandidateElement.GetClass(); + + // iterate through the sibling elements and decide whether append them + foreach (XElement siblingElement in siblingElements) + { + bool append = false; + string siblingElementName = GetElementName(siblingElement); + float contentBonus = 0; + + // Give a bonus if sibling nodes and top canidates have the same class name + if (!string.IsNullOrEmpty(topCandidateClass) && siblingElement.GetClass() == topCandidateClass) + { + contentBonus += topCandidateElementScore * _SiblingScoreTresholdCoefficient; + } + + if (siblingElement == topCandidateElement) + { + // we'll append the article content element (created from the top candidate element during an earlier step) + append = true; + } + else if ((GetElementScore(siblingElement) + contentBonus) >= siblingScoreThreshold) + { + // we'll append this element if the calculated score is higher than a treshold (derived from the score of the top candidate element) + append = true; + } + else if ("p".Equals(siblingElementName, StringComparison.OrdinalIgnoreCase)) + { + // we have to somehow decide whether we should append this paragraph + + string siblingElementInnerText = GetInnerText(siblingElement); + + // we won't append an empty paragraph + if (siblingElementInnerText.Length > 0) + { + int siblingElementInnerTextLength = siblingElementInnerText.Length; + + if (siblingElementInnerTextLength >= _MinSiblingParagraphLength) + { + // we'll append this paragraph if the links density is not higher than a treshold + append = GetLinksDensity(siblingElement) < _MaxSiblingParagraphLinksDensity; + } + else + { + // we'll append this paragraph if there are no links inside and if it contains a probable end of sentence indicator + append = GetLinksDensity(siblingElement).IsCloseToZero() + && _EndOfSentenceRegex.IsMatch(siblingElementInnerText); + } + } + } + + if (append) + { + XElement elementToAppend; + + if ("div".Equals(siblingElementName, StringComparison.OrdinalIgnoreCase) + || "p".Equals(siblingElementName, StringComparison.OrdinalIgnoreCase)) + { + elementToAppend = siblingElement; + } + else + { + /* We have an element that isn't a common block level element, like a form or td tag. + * Turn it into a div so it doesn't get filtered out later by accident. */ + + elementToAppend = new XElement("div"); + elementToAppend.SetId(siblingElement.GetId()); + elementToAppend.SetClass(siblingElement.GetClass()); + elementToAppend.Add(siblingElement.Nodes()); + } + + articleContentElement.Add(elementToAppend); + } + } + + return articleContentElement; + } + + internal void PrepareArticleContentElement(XElement articleContentElement) + { + CleanStyles(articleContentElement); + KillBreaks(articleContentElement); + + /* Clean out junk from the article content. */ + Clean(articleContentElement, "form"); + Clean(articleContentElement, "object"); + + if (articleContentElement.GetElementsByTagName("h1").Count() == 1) + { + Clean(articleContentElement, "h1"); + } + + /* If there is only one h2, they are probably using it as a header and not a subheader, + * so remove it since we already have a header. */ + if (articleContentElement.GetElementsByTagName("h2").Count() == 1) + { + Clean(articleContentElement, "h2"); + } + + Clean(articleContentElement, "iframe"); + CleanHeaders(articleContentElement); + + /* Do these last as the previous stuff may have removed junk that will affect these. */ + CleanConditionally(articleContentElement, "table"); + CleanConditionally(articleContentElement, "ul"); + CleanConditionally(articleContentElement, "div"); + + /* Remove extra paragraphs. */ + IEnumerable paraElements = articleContentElement.GetElementsByTagName("p"); + var elementsToRemove = new List(); + + foreach (XElement paraElement in paraElements) + { + string innerText = GetInnerText(paraElement, false); + if (innerText.Length > 0) { continue; } + + int imgsCount = paraElement.GetElementsByTagName("img").Count(); + if (imgsCount > 0) { continue; } + + int embedsCount = paraElement.GetElementsByTagName("embed").Count(); + if (embedsCount > 0) { continue; } + + int objectsCount = paraElement.GetElementsByTagName("object").Count(); + if (objectsCount > 0) { continue; } + + // We have a paragraph with empty inner text, with no images, no embeds and no objects. + // Let's remove it. + elementsToRemove.Add(paraElement); + } + + RemoveElements(elementsToRemove); + + /* Remove br's that are directly before paragraphs. */ + articleContentElement.SetInnerHtml(_BreakBeforeParagraphRegex.Replace(articleContentElement.GetInnerHtml(), " GetInnerText(anchorElement).Length); + + return (float)linksLength / elementInnerTextLength; + } + + internal int GetSegmentsCount(string s, char ch) + { + return s.Split(ch).Length; + } + + ///

+ /// Get "class/id weight" of the given . Uses regular expressions to tell if this element looks good or bad. + /// + internal int GetClassWeight(XElement element) + { + if (_dontWeightClasses) + { + return 0; + } + + int weight = 0; + + /* Look for a special classname. */ + string elementClass = element.GetClass(); + + if (elementClass.Length > 0) + { + if (_NegativeWeightRegex.IsMatch(elementClass)) + { + weight -= 25; + } + + if (_PositiveWeightRegex.IsMatch(elementClass)) + { + weight += 25; + } + } + + /* Look for a special ID */ + string elementId = element.GetId(); + + if (elementId.Length > 0) + { + if (_NegativeWeightRegex.IsMatch(elementId)) + { + weight -= 25; + } + + if (_PositiveWeightRegex.IsMatch(elementId)) + { + weight += 25; + } + } + + return weight; + } + + internal string GetInnerText(XNode node, bool dontNormalizeSpaces) + { + if (node == null) + { + throw new ArgumentNullException("node"); + } + + string result; + + if (node is XElement) + { + result = ((XElement)node).Value; + } + else if (node is XText) + { + result = ((XText)node).Value; + } + else + { + throw new NotSupportedException(string.Format("Nodes of type '{0}' are not supported.", node.GetType())); + } + + result = (result ?? "").Trim(); + + if (!dontNormalizeSpaces) + { + return _NormalizeSpacesRegex.Replace(result, " "); + } + + return result; + } + + internal string GetInnerText(XNode node) + { + return GetInnerText(node, _dontNormalizeSpacesInTextContent); + } + + /// + /// Removes extraneous break tags from a . + /// + internal void KillBreaks(XElement element) + { + element.SetInnerHtml(_KillBreaksRegex.Replace(element.GetInnerHtml(), "
")); + } + + /// + /// Cleans an element of all elements with name . + /// (Unless it's a youtube/vimeo video. People love movies.) + /// + internal void Clean(XElement rootElement, string elementName) + { + IEnumerable elements = rootElement.GetElementsByTagName(elementName); + + bool isEmbed = "object".Equals(elementName, StringComparison.OrdinalIgnoreCase) + || "embed".Equals(elementName, StringComparison.OrdinalIgnoreCase); + + var elementsToRemove = new List(); + + foreach (XElement element in elements) + { + /* Allow youtube and vimeo videos through as people usually want to see those. */ + if (isEmbed + && (_VideoRegex.IsMatch(element.GetAttributesString("|")) + || _VideoRegex.IsMatch(element.GetInnerHtml()))) + { + continue; + } + + elementsToRemove.Add(element); + } + + RemoveElements(elementsToRemove); + } + + /// + /// Cleans a of all elements with name if they look fishy. + /// "Fishy" is an algorithm based on content length, classnames, link density, number of images and embeds, etc. + /// + internal void CleanConditionally(XElement rootElement, string elementName) + { + if (elementName == null) + { + throw new ArgumentNullException("elementName"); + } + + IEnumerable elements = rootElement.GetElementsByTagName(elementName); + var elementsToRemove = new List(); + + foreach (XElement element in elements) + { + int weight = GetClassWeight(element); + float score = GetElementScore(element); + + if (weight + score < 0.0f) + { + elementsToRemove.Add(element); + continue; + } + + if (ElementLooksLikeParagraphDiv(element)) + { + // leave the element - it's probably just a div pretending to be a paragraph + continue; + } + + /* If there are not very many commas and the number of non-paragraph elements + * is more than paragraphs or other ominous signs, remove the element. */ + string elementInnerText = GetInnerText(element); + + if (GetSegmentsCount(elementInnerText, ',') < _MinCommaSegments) + { + int psCount = element.GetElementsByTagName("p").Count(); + int imgsCount = element.GetElementsByTagName("img").Count(); + int lisCount = element.GetElementsByTagName("li").Count(); + int inputsCount = element.GetElementsByTagName("input").Count(); + + // while counting embeds we omit video-embeds + int embedsCount = + element.GetElementsByTagName("embed") + .Count(embedElement => !_VideoRegex.IsMatch(embedElement.GetAttributeValue("src", ""))); + + float linksDensity = GetLinksDensity(element); + int innerTextLength = elementInnerText.Length; + string elementNameLower = elementName.Trim().ToLower(); + bool remove = (imgsCount > psCount) + || (lisCount - _LisCountTreshold > psCount && elementNameLower != "ul" && elementNameLower != "ol") + || (inputsCount > psCount / 3) + || (innerTextLength < _MinInnerTextLength && (imgsCount == 0 || imgsCount > _MaxImagesInShortSegmentsCount)) + || (weight < _ClassWeightTreshold && linksDensity > _MaxDensityForElementsWithSmallerClassWeight) + || (weight >= _ClassWeightTreshold && linksDensity > _MaxDensityForElementsWithGreaterClassWeight) + || (embedsCount > _MaxEmbedsCount || (embedsCount == _MaxEmbedsCount && innerTextLength < _MinInnerTextLengthInElementsWithEmbed)); + + if (remove) + { + elementsToRemove.Add(element); + } + + } + } /* end foreach */ + + RemoveElements(elementsToRemove); + } + + /// + /// Cleans out spurious headers from a . Checks things like classnames and link density. + /// + internal void CleanHeaders(XElement element) + { + var elementsToRemove = new List(); + + for (int headerLevel = 1; headerLevel < 7; headerLevel++) + { + IEnumerable headerElements = element.GetElementsByTagName("h" + headerLevel); + + foreach (XElement headerElement in headerElements) + { + if (GetClassWeight(headerElement) < 0 + || GetLinksDensity(headerElement) > _MaxHeaderLinksDensity) + { + elementsToRemove.Add(headerElement); + } + } + } + + RemoveElements(elementsToRemove); + } + + /// + /// Removes the style attribute from the specified and all elements underneath it. + /// + internal void CleanStyles(XElement rootElement) + { + new ElementsTraverser( + element => + { + string elementClass = element.GetClass(); + + if (elementClass.Contains(ReadabilityStyledCssClass)) + { + // don't remove the style if that's we who have styled this element + return; + } + + element.SetStyle(null); + }).Traverse(rootElement); + } + + internal string GetUserStyleClass(string prefix, String enumStr) + { + var suffixSB = new StringBuilder(); + bool wasUpperCaseCharacterSeen = false; + + enumStr.ToCharArray().Aggregate( + suffixSB, + (sb, ch) => + { + if (Char.IsUpper(ch)) + { + if (wasUpperCaseCharacterSeen) + { + sb.Append('-'); + } + + wasUpperCaseCharacterSeen = true; + + sb.Append(Char.ToLower(ch)); + } + else + { + sb.Append(ch); + } + + return sb; + }); + + return string.Format("{0}-{1}", prefix, suffixSB).TrimEnd('-'); + } + + #endregion + + #region Private helper methods + + private static XElement GetOrCreateBody(XDocument document) + { + XElement documentBody = document.GetBody(); + + if (documentBody == null) + { + XElement htmlElement = document.GetChildrenByTagName("html").FirstOrDefault(); + + if (htmlElement == null) + { + htmlElement = new XElement("html"); + document.Add(htmlElement); + } + + documentBody = new XElement("body"); + htmlElement.Add(documentBody); + } + + return documentBody; + } + + private static void RemoveElements(IEnumerable elementsToRemove) + { + elementsToRemove.ForEach(elementToRemove => elementToRemove.Remove()); + } + + private static void ResolveElementsUrls(XDocument document, string tagName, string attributeName, string url, Func attributeValueTransformer) + { + if (document == null) + { + throw new ArgumentNullException("document"); + } + + if (string.IsNullOrEmpty(url)) + { + throw new ArgumentNullException("url"); + } + + IEnumerable elements = document.GetElementsByTagName(tagName); + + foreach (XElement element in elements) + { + string attributeValue = element.GetAttributeValue(attributeName, null); + + if (attributeValue == null) + { + continue; + } + + attributeValue = ResolveElementUrl(attributeValue, url); + + if (!string.IsNullOrEmpty(attributeValue)) + { + AttributeTransformationResult attributeTransformationResult; + + if (attributeValueTransformer != null) + { + attributeTransformationResult = attributeValueTransformer.Invoke(new AttributeTransformationInput { AttributeValue = attributeValue, Element = element }); + } + else + { + attributeTransformationResult = new AttributeTransformationResult { TransformedValue = attributeValue }; + } + + element.SetAttributeValue(attributeName, attributeTransformationResult.TransformedValue); + + if (!string.IsNullOrEmpty(attributeTransformationResult.OriginalValueAttributeName)) + { + element.SetAttributeValue(attributeTransformationResult.OriginalValueAttributeName, attributeValue); + } + } + } + } + + private static string ResolveElementUrl(string url, string articleUrl) + { + if (url == null) + { + throw new ArgumentNullException(); + } + + if (_MailtoHrefRegex.IsMatch(url)) + { + return url; + } + + Uri baseUri; + + if (!Uri.TryCreate(articleUrl, UriKind.Absolute, out baseUri)) + { + return url; + } + + /* If the link is simply a query string, then simply attach it to the original URL */ + if (url.StartsWith("?")) + { + return baseUri.Scheme + "://" + baseUri.Host + baseUri.AbsolutePath + url; + } + + Uri absoluteUri; + + if (Uri.TryCreate(baseUri, url, out absoluteUri)) + { + return absoluteUri.OriginalString; + } + + return url; + } + + private static string GetElementName(XElement element) + { + return element.Name.LocalName ?? ""; + } + + private static void SetElementName(XElement element, string newLocalName) + { + element.Name = XName.Get(newLocalName, element.Name.NamespaceName); + } + + private static bool ElementLooksLikeParagraphDiv(XElement element) + { + string elementName = GetElementName(element); + + if (!"div".Equals(elementName, StringComparison.OrdinalIgnoreCase)) + { + return false; + } + + if (!_LikelyParagraphDivRegex.IsMatch(element.GetClass())) + { + // we'll consider divs only with certain classes as potential paragraph divs + return false; + } + + XNode childNode = element.Nodes().SingleOrNone(); + + if (childNode != null) + { + XElement childElement = childNode as XElement; + + if (childElement != null + && "p".Equals(GetElementName(childElement), StringComparison.OrdinalIgnoreCase)) + { + // we have a div with a single child element that is a paragraph + return true; + } + } + + return false; + } + + private static string ExtractTitle(XDocument transcodedXmlDocument) + { + XElement firstH1Element = + transcodedXmlDocument.Root + .GetElementsByTagName("h1").FirstOrDefault(); + + string extractedTitle = + firstH1Element != null + ? firstH1Element.Value + : null; + + if (!string.IsNullOrEmpty(extractedTitle)) + { + extractedTitle = _TitleWhitespacesCleanUpRegex.Replace(extractedTitle, " "); + extractedTitle = extractedTitle.Trim(); + } + + if (extractedTitle != null && extractedTitle.Length == 0) + { + extractedTitle = null; + } + + return extractedTitle; + } + + private static XElement TryFindArticleContentElement(XDocument document, string articleContentElementHint) + { + if (document == null) + { + throw new ArgumentNullException("document"); + } + + if (string.IsNullOrEmpty(articleContentElementHint)) + { + throw new ArgumentException("Argument can't be null nor empty.", "articleContentElementHint"); + } + + return document + .GetElementsByTagName(articleContentElementHint) + .FirstOrDefault(); + } + + private static string GetArticleContentElementHint(string url) + { + if (string.IsNullOrEmpty(url)) + { + throw new ArgumentException("Argument can't be null nor empty.", "url"); + } + + url = url.Trim(); + + foreach (Regex urlRegex in _articleContentElementHints.Keys) + { + if (urlRegex.IsMatch(url)) + { + return _articleContentElementHints[urlRegex]; + } + } + + return null; + } + + private string GetReadingStyleClass(ReadingStyle readingStyle) + { + return GetUserStyleClass("style", readingStyle.ToString()); + } + + private string GetReadingMarginClass(ReadingMargin readingMargin) + { + return GetUserStyleClass("margin", readingMargin.ToString()); + } + + private string GetReadingSizeClass(ReadingSize readingSize) + { + return GetUserStyleClass("size", readingSize.ToString()); + } + + private void AddPointsToElementScore(XElement element, int pointsToAdd) + { + float currentScore = _elementsScores.ContainsKey(element) ? _elementsScores[element] : 0.0f; + + _elementsScores[element] = currentScore + pointsToAdd; + } + + private float GetElementScore(XElement element) + { + return _elementsScores.ContainsKey(element) ? _elementsScores[element] : 0.0f; + } + + private void SetElementScore(XElement element, float score) + { + _elementsScores[element] = score; + } + + #endregion + + #region Properties + + /// + /// A function to transform the value of 'src' attribute on 'img' elements. Can be null. + /// + public Func ImageSourceTranformer + { + get { return _imageSourceTranformer; } + set { _imageSourceTranformer = value; } + } + + /// + /// A function to transform the value of 'href' attribute on 'a' elements. Can be null. + /// + public Func AnchorHrefTranformer + { + get { return _anchorHrefTransformer; } + set { _anchorHrefTransformer = value; } + } + + #endregion + } +} diff --git a/PortablePorts/NReadability/Properties/AssemblyInfo.cs b/PortablePorts/NReadability/Properties/AssemblyInfo.cs new file mode 100644 index 0000000..913278d --- /dev/null +++ b/PortablePorts/NReadability/Properties/AssemblyInfo.cs @@ -0,0 +1,30 @@ +using System.Resources; +using System.Reflection; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; + +// General Information about an assembly is controlled through the following +// set of attributes. Change these attribute values to modify the information +// associated with an assembly. +[assembly: AssemblyTitle("NReadabilityPCL")] +[assembly: AssemblyDescription("")] +[assembly: AssemblyConfiguration("")] +[assembly: AssemblyCompany("")] +[assembly: AssemblyProduct("NReadabilityPCL")] +[assembly: AssemblyCopyright("Copyright © 2013")] +[assembly: AssemblyTrademark("")] +[assembly: AssemblyCulture("")] +[assembly: NeutralResourcesLanguage("en")] + +// Version information for an assembly consists of the following four values: +// +// Major Version +// Minor Version +// Build Number +// Revision +// +// You can specify all the values or you can default the Build and Revision Numbers +// by using the '*' as shown below: +// [assembly: AssemblyVersion("1.0.*")] +[assembly: AssemblyVersion("1.0.0.0")] +[assembly: AssemblyFileVersion("1.0.0.0")] diff --git a/PortablePorts/NReadability/Resources/readability.css b/PortablePorts/NReadability/Resources/readability.css new file mode 100644 index 0000000..af90293 --- /dev/null +++ b/PortablePorts/NReadability/Resources/readability.css @@ -0,0 +1,82 @@ +/* Document */ +body {font-size: 100%;} +#readability-logo,#arc90-logo,.footer-twitterLink,#readTools a,a.rdbTK-powered span{background-color:transparent!important;background-image:url(http://lab.arc90.com/experiments/readability/images/sprite-readability.png)!important;background-repeat:no-repeat!important;} +#readOverlay {display:block;position:absolute;top:0;left:0;width:100%;} +#readInner {line-height:1.4em;max-width:800px;margin:1em auto;} +#readInner a {color:#039;text-decoration:none;} +#readInner a:hover {text-decoration:underline;} +#readInner img {float:left;clear:both;margin: 0 12px 12px 0;} +#readInner h1 {display:block;width:100%;border-bottom:1px solid #333;font-size:1.2em;padding-bottom:.5em;margin-top:0;margin-bottom:.75em;} +#readInner sup{line-height:.8em;} +#readInner .page-separator{clear:both;display:block;font-size:.85em;filter:alpha(opacity=20);opacity:.20;text-align:center;} +.style-apertura #readInner h1 {border-bottom-color:#ededed;} +#readInner blockquote {margin-left:3em;margin-right:3em;} +#readability-inner * {margin-bottom:16px;border:none;background:none;} +/* Footer */ +#readFooter {display:block;border-top:1px solid #333;text-align:center;clear:both;overflow:hidden;} +.style-apertura #readFooter {border-top-color:#ededed;} +#rdb-footer-left {display:inline;float:left;margin-top:15px;width:285px;background-position:0 -36px;} +.rdbTypekit #rdb-footer-left {width:475px;} +#rdb-footer-left a,#rdb-footer-left a:link {float:left;} +#readability-logo {display:inline;background-position:0 -36px;height:29px;width:189px;text-indent:-9000px;} +#arc90-logo {display:inline;background-position:right -36px;height:29px;width:96px;text-indent:-9000px;} +#readability-url {display:none;} +.style-apertura #readability-logo {background-position:0 -67px;} +.style-apertura #arc90-logo {background-position:right -67px;} +#rdb-footer-right {display:inline;float:right;text-align:right;font-size:.75em;margin-top:18px;} +#rdb-footer-right a {display:inline-block;float:left;overflow:visible;line-height:16px;vertical-align:baseline;} +.footer-twitterLink {height:20px;margin-left:20px;padding:4px 0 0 28px;background-position:0 -123px;font-size:12px;} +#rdb-footer-left .footer-twitterLink {display:none;margin-top:1px;padding-top:2px;} +.rdbTypekit #rdb-footer-right .footer-twitterLink {display:none;} +.rdbTypekit #rdb-footer-left .footer-twitterLink {display:inline-block!important;} +a.rdbTK-powered,a.rdbTK-powered:link,a.rdbTK-powered:hover {font-size:16px;color:#858789!important;text-decoration:none!important;} +a.rdbTK-powered span {display:inline-block;height:22px;margin-left:2px;padding:4px 0 0 26px;background-position:0 -146px!important;} +.style-apertura #rdb-inverse,.style-athelas #rdb-athelas {display:block;} +span.version {display:none;} +/* Tools */ +#readTools {width:34px;height:150px;position:fixed;z-index:100;top:10px;left:10px;} +#readTools a {overflow:hidden;margin-bottom:8px;display:block;opacity:.4;text-indent:-99999px;height:34px;width:34px;text-decoration:none;filter:alpha(opacity=40);} +#reload-page {background-position:0 0;} +#print-page {background-position:-36px 0;} +#email-page {background-position:-72px 0;} +#kindle-page {background-position:-108px 0;} +#readTools a:hover {opacity:1;filter:alpha(opacity=100);} +/* -- USER-CONFIGURABLE STYLING -- */ +/* Size */ +.size-x-small {font-size:.75em;} +.size-small {font-size:.938em;} +.size-medium {font-size:1.125em;} +.size-large {font-size:1.375em;} +.size-x-large {font-size:1.75em;} +/* Style */ +.style-newspaper {font-family:"Times New Roman", Times, serif;background:#fbfbfb;color:#080000;} +.style-newspaper h1 {text-transform:capitalize;font-family:Georgia, "Times New Roman", Times, serif;} +.style-newspaper #readInner a {color:#0924e1;} +.style-novel {font-family:"Palatino Linotype", "Book Antiqua", Palatino, serif;background:#f4eed9;color:#1d1916;} +.style-novel #readInner a {color:#1856ba;} +.style-ebook {font-family:Arial, Helvetica, sans-serif;background:#edebe8;color:#2c2d32;} +.style-ebook #readInner a {color:#187dc9;} +.style-ebook h1 {font-family:"Arial Black", Gadget, sans-serif;font-weight:400;} +.style-terminal {font-family:"Lucida Console", Monaco, monospace;background:#1d4e2c;color:#c6ffc6;} +.style-terminal #readInner a {color:#093;} +/* Typekit */ +.style-apertura {font-family:"apertura-1", "apertura-2", sans-serif;background-color:#2d2828;color:#eae8e9;} +.style-apertura #readInner a {color:#58b0ff;} +.style-athelas {font-family:"athelas-1", "athelas-2", "Palatino Linotype", "Book Antiqua", Palatino, serif;background-color:#f7f7f7;color:#2b373d;} +.style-athelas #readInner a {color:#1e83cb;} +/* Margin */ +.margin-x-narrow {width:95%;} +.margin-narrow {width:85%;} +.margin-medium {width:75%;} +.margin-wide {width:55%;} +.margin-x-wide {width:35%;} +/* -- USER-CONFIGURABLE STYLING -- */ +/* -- DEBUG -- */ +.bug-green {background:#bbf9b0;border:4px solid green;} +.bug-red {background:red;} +.bug-yellow {background:#ffff8e;} +.bug-blue {background:#bfdfff;} +/* -- EMAIL / KINDLE POP UP -- */ +#kindle-container, #email-container {position:fixed;top:60px;left:50%;width:500px;height:490px;border:solid 3px #666;background-color:#fff;z-index:100!important;overflow:hidden;margin:0 0 0 -240px;padding:0;} +/* Override html styling attributes */ +table, tr, td { background-color: transparent !important; } diff --git a/PortablePorts/NReadability/SgmlDomBuilder.cs b/PortablePorts/NReadability/SgmlDomBuilder.cs new file mode 100644 index 0000000..b5454d2 --- /dev/null +++ b/PortablePorts/NReadability/SgmlDomBuilder.cs @@ -0,0 +1,114 @@ +/* + * NReadability + * http://code.google.com/p/nreadability/ + * + * Copyright 2010 Marek Stój + * http://immortal.pl/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System; +using System.Text; +using System.Text.RegularExpressions; +using System.Xml; +using System.Xml.Linq; +using ReadSharp.Ports.Sgml; +using System.IO; + +namespace ReadSharp.Ports.NReadability +{ + /// + /// A class for constructing a DOM from HTML markup. + /// + public class SgmlDomBuilder + { + #region Public methods + + /// + /// Constructs a DOM (System.Xml.Linq.XDocument) from HTML markup. + /// + /// HTML markup from which the DOM is to be constructed. + /// System.Linq.Xml.XDocument instance which is a DOM of the provided HTML markup. + public XDocument BuildDocument(string htmlContent) + { + if (htmlContent == null) + { + throw new ArgumentNullException("htmlContent"); + } + + if (htmlContent.Trim().Length == 0) + { + return new XDocument(); + } + + // "trim end" htmlContent to ...$ (codinghorror.com puts some scripts after the - sic!) + const string htmlEnd = "', indexOfHtmlEnd); + + if (indexOfHtmlEndBracket != -1) + { + htmlContent = htmlContent.Substring(0, indexOfHtmlEndBracket + 1); + } + } + + XDocument document; + + try + { + document = LoadDocument(htmlContent); + } + catch (InvalidOperationException exc) + { + // sometimes SgmlReader doesn't handle