ReadSharp/PortablePorts/NReadability/NReadabilityTranscoder.cs

/*
 * NReadability
 * http://code.google.com/p/nreadability/
 * 
 * Copyright 2010 Marek Stój
 * http://immortal.pl/
 * 
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 * 
 * http://www.apache.org/licenses/LICENSE-2.0
 * 
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Reflection;
using System.Text;
using System.Text.RegularExpressions;
using System.Xml;
using System.Xml.Linq;

namespace ReadSharp.Ports.NReadability
{
  /// <summary>
  /// A class that extracts main content from a HTML page.
  /// </summary>
  public class NReadabilityTranscoder
  {
    #region Nested types

    /// <summary>
    /// Used in FindNextPageLink
    /// </summary>
    private class LinkData
    {
      public float Score;
      public string LinkText;
      public string LinkHref;
    }

    #endregion

    private static readonly string _ReadabilityStylesheetResourceName = typeof(NReadabilityTranscoder).Namespace + ".Resources.readability.css";

    #region Algorithm constants

    /// <summary>
    /// Default styling of the extracted article.
    /// </summary>
    public const ReadingStyle DefaultReadingStyle = ReadingStyle.Newspaper;

    /// <summary>
    /// Default margin of the extracted article.
    /// </summary>
    public const ReadingMargin DefaultReadingMargin = ReadingMargin.Wide;

    /// <summary>
    /// Default size of the font used for the extracted article.
    /// </summary>
    public const ReadingSize DefaultReadingSize = ReadingSize.Medium;

    internal const string OverlayDivId = "readOverlay";
    internal const string InnerDivId = "readInner";
    internal const string ContentDivId = "readability-content";
    internal const string ReadabilityStyledCssClass = "readability-styled";

    private const int _MinParagraphLength = 25;
    private const int _MinInnerTextLength = 25;
    private const int _ParagraphSegmentLength = 100;
    private const int _MaxPointsForSegmentsCount = 3;
    private const int _MinSiblingParagraphLength = 80;
    private const int _MinCommaSegments = 10;
    private const int _LisCountTreshold = 100;
    private const int _MaxImagesInShortSegmentsCount = 2;
    private const int _MinInnerTextLengthInElementsWithEmbed = 75;
    private const int _ClassWeightTreshold = 25;
    private const int _MaxEmbedsCount = 1;
    private const int _MaxArticleTitleLength = 150;
    private const int _MinArticleTitleLength = 15;
    private const int _MinArticleTitleWordsCount1 = 3;
    private const int _MinArticleTitleWordsCount2 = 4;

    private const float _SiblingScoreTresholdCoefficient = 0.2f;
    private const float _MaxSiblingScoreTreshold = 10.0f;
    private const float _MaxSiblingParagraphLinksDensity = 0.25f;
    private const float _MaxHeaderLinksDensity = 0.33f;
    private const float _MaxDensityForElementsWithSmallerClassWeight = 0.2f;
    private const float _MaxDensityForElementsWithGreaterClassWeight = 0.5f;

    #endregion

    #region Algorithm regular expressions

    private static readonly Regex _UnlikelyCandidatesRegex = new Regex("combx|comment|community|disqus|extra|foot|header|menu|remark|rss|shoutbox|sidebar|side|sponsor|ad-break|agegate|pagination|pager|popup|tweet|twitter", RegexOptions.IgnoreCase);
    private static readonly Regex _OkMaybeItsACandidateRegex = new Regex("and|article|body|column|main|shadow", RegexOptions.IgnoreCase);
    private static readonly Regex _PositiveWeightRegex = new Regex("article|body|content|entry|hentry|main|page|pagination|post|text|blog|story|paragraph|instapaper_body|entry-content|article-body|entry-body", RegexOptions.IgnoreCase);
    private static readonly Regex _NegativeWeightRegex = new Regex("combx|comment|com-|contact|foot|footer|footnote|masthead|media|meta|outbrain|promo|related|scroll|shoutbox|sidebar|side|sponsor|shopping|tags|tool|widget", RegexOptions.IgnoreCase);
    private static readonly Regex _NegativeLinkParentRegex = new Regex("(stories|articles|news|documents|posts|notes|series|historie|artykuly|artykuły|wpisy|dokumenty|serie|geschichten|erzählungen|erzahlungen)", RegexOptions.IgnoreCase);
    private static readonly Regex _PositiveNavPaginationRegex = new Regex("pagination|pages|numbers|seite|page", RegexOptions.IgnoreCase);
    private static readonly Regex _Extraneous = new Regex("print|archive|comment|discuss|e[-]?mail|share|reply|all|login|sign|single|also", RegexOptions.IgnoreCase);
    private static readonly Regex _DivToPElementsRegex = new Regex("<(a|blockquote|dl|div|img|ol|p|pre|table|ul)", RegexOptions.IgnoreCase);
    private static readonly Regex _EndOfSentenceRegex = new Regex("\\.( |$)", RegexOptions.Multiline);
    private static readonly Regex _BreakBeforeParagraphRegex = new Regex("<br[^>]*>\\s*<p");
    private static readonly Regex _NormalizeSpacesRegex = new Regex("\\s{2,}");
    private static readonly Regex _KillBreaksRegex = new Regex("(<br\\s*\\/?>(\\s|&nbsp;?)*){1,}");
    private static readonly Regex _VideoRegex = new Regex("\\/\\/(www\\.)?(youtube|vimeo)\\.com", RegexOptions.IgnoreCase);
    private static readonly Regex _ReplaceDoubleBrsRegex = new Regex("(<br[^>]*>[ \\n\\r\\t]*){2,}", RegexOptions.IgnoreCase);
    private static readonly Regex _ReplaceFontsRegex = new Regex("<(\\/?)font[^>]*>", RegexOptions.IgnoreCase);
    private static readonly Regex _ArticleTitleDashRegex1 = new Regex(" [\\|\\-] ");
    private static readonly Regex _ArticleTitleDashRegex2 = new Regex("(.*)[\\|\\-] .*");
    private static readonly Regex _ArticleTitleDashRegex3 = new Regex("[^\\|\\-]*[\\|\\-](.*)");
    private static readonly Regex _ArticleTitleColonRegex1 = new Regex(".*:(.*)");
    private static readonly Regex _ArticleTitleColonRegex2 = new Regex("[^:]*[:](.*)");
    private static readonly Regex _NextLink = new Regex(@"(next|weiter|nächste|nächstes|^([0-9]\.)|continue|dalej|następna|nastepna>([^\|]|$)|�([^\|]|$))", RegexOptions.IgnoreCase);
    private static readonly Regex _NextStoryLink = new Regex("(story|article|news|document|post|note|series|historia|artykul|artykuł|wpis|dokument|seria|geschichte|erzählung|erzahlung|artikel|serie)", RegexOptions.IgnoreCase);
    private static readonly Regex _PrevLink = new Regex("(prev|earl|[^b]old|new|wstecz|poprzednia|<|�)", RegexOptions.IgnoreCase);
    private static readonly Regex _PageRegex = new Regex("pag(e|ing|inat)|([^a-z]|^)pag([^a-z]|$)", RegexOptions.IgnoreCase);
    private static readonly Regex _LikelyParagraphDivRegex = new Regex("text|para|parbase|paragraph|figure", RegexOptions.IgnoreCase);
    private static readonly Regex _LikelyImageContainerRegex = new Regex("image|figure|photo|media", RegexOptions.IgnoreCase);
    private static readonly Regex _ReplaceAttributes = new Regex("^(data-|aria-|name|height|width|item|value|contenteditable|on|target|type|align|direction)", RegexOptions.IgnoreCase);

    #endregion

    #region Other regular expressions

    private static readonly Regex _MailtoHrefRegex = new Regex("^\\s*mailto\\s*:", RegexOptions.IgnoreCase);
    private static readonly Regex _TitleWhitespacesCleanUpRegex = new Regex("\\s+");

    private static readonly Dictionary<Regex, string> _articleContentElementHints = new Dictionary<Regex, string>
    {
      { new Regex("^https?://(www|mobile)\\.theverge.com", RegexOptions.IgnoreCase), ".entry-body" },
      { new Regex("^https?://(www|blog)\\.bufferapp.com", RegexOptions.IgnoreCase), ".post" },
      { new Regex("^https?://(www.)?polygon.com", RegexOptions.IgnoreCase), ".body" },
      { new Regex("^https?://(www.)?medium.com", RegexOptions.IgnoreCase), ".body" },
      { new Regex("^https?://(www.)?it-scoop.com", RegexOptions.IgnoreCase), ".entry-content" },
      { new Regex("^https?://(m\\.|www\\.)?youtube.com", RegexOptions.IgnoreCase), "#watch-description-content" },
      { new Regex("^https?://(www.)?vimeo.com", RegexOptions.IgnoreCase), ".description_wrapper" },
      { new Regex("^https?://(www.)?zdnet.com", RegexOptions.IgnoreCase), ".storyBody" },
      { new Regex("^https?://(www.)?recode.net", RegexOptions.IgnoreCase), ".postarea" }
    };

    #endregion

    #region Algorithm parameters

    private bool _dontStripUnlikelys;
    private readonly bool _dontNormalizeSpacesInTextContent;
    private readonly bool _dontWeightClasses;
    private readonly ReadingStyle _readingStyle;
    private readonly ReadingSize _readingSize;
    private readonly ReadingMargin _readingMargin;

    #endregion

    #region Helper instance fields

    private readonly SgmlDomBuilder _sgmlDomBuilder;
    private readonly SgmlDomSerializer _sgmlDomSerializer;
    private readonly Dictionary<XElement, float> _elementsScores;

    private Func<AttributeTransformationInput, AttributeTransformationResult> _imageSourceTranformer;
    private Func<AttributeTransformationInput, AttributeTransformationResult> _anchorHrefTransformer;
    private Func<AttributeTransformationInput, AttributeTransformationResult> _metaTransformer;

    #endregion

    #region Constructor(s)

    /// <summary>
    /// Initializes a new instance of NReadabilityTranscoder. Allows setting all options.
    /// </summary>
    /// <param name="dontStripUnlikelys">Determines whether elements that are unlikely to be a part of main content will be removed.</param>
    /// <param name="dontNormalizeSpacesInTextContent">Determines whether spaces in InnerText properties of elements will be normalized automatically (eg. whether double spaces will be replaced with single spaces).</param>
    /// <param name="dontWeightClasses">Determines whether 'weight-class' algorithm will be used when cleaning content.</param>
    /// <param name="readingStyle">Styling for the extracted article.</param>
    /// <param name="readingMargin">Margin for the extracted article.</param>
    /// <param name="readingSize">Font size for the extracted article.</param>
    public NReadabilityTranscoder(
      bool dontStripUnlikelys,
      bool dontNormalizeSpacesInTextContent,
      bool dontWeightClasses,
      ReadingStyle readingStyle,
      ReadingMargin readingMargin,
      ReadingSize readingSize)
    {
      _dontStripUnlikelys = dontStripUnlikelys;
      _dontNormalizeSpacesInTextContent = dontNormalizeSpacesInTextContent;
      _dontWeightClasses = dontWeightClasses;
      _readingStyle = readingStyle;
      _readingMargin = readingMargin;
      _readingSize = readingSize;

      _sgmlDomBuilder = new SgmlDomBuilder();
      _sgmlDomSerializer = new SgmlDomSerializer();
      _elementsScores = new Dictionary<XElement, float>();
    }

    /// <summary>
    /// Initializes a new instance of NReadabilityTranscoder. Allows setting reading options.
    /// </summary>
    /// <param name="readingStyle">Styling for the extracted article.</param>
    /// <param name="readingMargin">Margin for the extracted article.</param>
    /// <param name="readingSize">Font size for the extracted article.</param>
    public NReadabilityTranscoder(ReadingStyle readingStyle, ReadingMargin readingMargin, ReadingSize readingSize)
      : this(false, false, false, readingStyle, readingMargin, readingSize)
    {
    }

    /// <summary>
    /// Initializes a new instance of NReadabilityTranscoder.
    /// </summary>
    public NReadabilityTranscoder()
      : this(DefaultReadingStyle, DefaultReadingMargin, DefaultReadingSize)
    {
    }

    #endregion

    #region Public methods

    /// <summary>
    /// Extracts article content from an HTML page.
    /// </summary>
    /// <param name="transcodingInput">An object containing input parameters, i.a. html content to be processed.</param>
    /// <returns>An object containing transcoding result, i.a. extracted content and title.</returns>
    public TranscodingResult Transcode(TranscodingInput transcodingInput)
    {
      if (transcodingInput == null)
      {
        throw new ArgumentNullException("transcodingInput");
      }

      bool contentExtracted;
      string extractedTitle;
      string nextPageUrl;

      XDocument transcodedXmlDocument =
        TranscodeToXml(
          transcodingInput.HtmlContent,
          transcodingInput.Url,
          out contentExtracted,
          out extractedTitle,
          out nextPageUrl);

      IEnumerable<XElement> images = transcodedXmlDocument.GetElementsByTagName("img");

      string transcodedContent =
        _sgmlDomSerializer.SerializeDocument(
          transcodedXmlDocument,
          transcodingInput.DomSerializationParams);

      bool titleExtracted = !string.IsNullOrEmpty(extractedTitle);

      MetaExtractor metaExtractor = new MetaExtractor(transcodedXmlDocument);

      string charset = null;
      string description = null;
      Uri image = null;
      Uri favicon = null;

      if (metaExtractor.HasValue)
      {
        charset = metaExtractor.GetCharset();
        description = metaExtractor.GetMetaDescription();
        string imageString = metaExtractor.GetMetaImage();
        string faviconString = metaExtractor.GetMetaFavicon();

        if (imageString != null)
        {
          imageString = ResolveElementUrl(imageString, transcodingInput.Url);
          Uri.TryCreate(imageString, UriKind.Absolute, out image);
        }
        if (faviconString != null)
        {
          faviconString = ResolveElementUrl(faviconString, transcodingInput.Url);
          Uri.TryCreate(faviconString, UriKind.Absolute, out favicon);
        }
      }

      return
        new TranscodingResult(contentExtracted, titleExtracted)
          {
            ExtractedContent = transcodedContent,
            ExtractedTitle = extractedTitle,
            ExtractedDescription = description,
            ExtractedFavicon = favicon,
            ExtractedImage = image,
            NextPageUrl = nextPageUrl,
            Charset = charset,
            Images = images
          };
    }

    /// <summary>
    /// Extracts main article content from a HTML page.
    /// </summary>
    /// <param name="htmlContent">HTML markup to process.</param>
    /// <param name="url">Url from which the content was downloaded. Used to resolve relative urls. Can be null.</param>
    /// <param name="domSerializationParams">Contains parameters that modify the behaviour of the output serialization.</param>
    /// <param name="mainContentExtracted">Determines whether the content has been extracted (if the article is not empty).</param>
    /// <param name="nextPageUrl">If the content contains a link to a subsequent page, it is returned here.</param>
    /// <returns>HTML markup containing extracted article content.</returns>
    [Obsolete("Use TranscodingResult Transcode(TranscodingInput) method.")]
    public string Transcode(string htmlContent, string url, DomSerializationParams domSerializationParams, out bool mainContentExtracted, out string nextPageUrl)
    {
      string extractedTitle;

      XDocument document =
        TranscodeToXml(
          htmlContent,
          url,
          out mainContentExtracted,
          out extractedTitle,
          out nextPageUrl);

      return _sgmlDomSerializer.SerializeDocument(document, domSerializationParams);
    }

    /// <summary>
    /// Extracts main article content from a HTML page.
    /// </summary>
    /// <param name="htmlContent">HTML markup to process.</param>
    /// <param name="url">Url from which the content was downloaded. Used to resolve relative urls. Can be null.</param>
    /// <param name="mainContentExtracted">Determines whether the content has been extracted (if the article is not empty).</param>
    /// <param name="nextPageUrl">If the content contains a link to a subsequent page, it is returned here.</param>
    /// <returns>HTML markup containing extracted article content.</returns>
    [Obsolete("Use TranscodingResult Transcode(TranscodingInput) method.")]
    public string Transcode(string htmlContent, string url, out bool mainContentExtracted, out string nextPageUrl)
    {
      return Transcode(htmlContent, url, DomSerializationParams.CreateDefault(), out mainContentExtracted, out nextPageUrl);
    }

    /// <summary>
    /// Extracts main article content from a HTML page.
    /// </summary>
    /// <param name="htmlContent">HTML markup to process.</param>
    /// <param name="url">Url from which the content was downloaded. Used to resolve relative urls. Can be null.</param>
    /// <param name="mainContentExtracted">Determines whether the content has been extracted (if the article is not empty).</param>    
    /// <returns>HTML markup containing extracted article content.</returns>
    [Obsolete("Use TranscodingResult Transcode(TranscodingInput) method.")]
    public string Transcode(string htmlContent, string url, out bool mainContentExtracted)
    {
      string nextPageUrl;

      return Transcode(htmlContent, url, DomSerializationParams.CreateDefault(), out mainContentExtracted, out nextPageUrl);
    }

    /// <summary>
    /// Extracts main article content from a HTML page.
    /// </summary>
    /// <param name="htmlContent">HTML markup to process.</param>
    /// <param name="mainContentExtracted">Determines whether the content has been extracted (if the article is not empty).</param>    
    /// <returns>HTML markup containing extracted article content.</returns>
    [Obsolete("Use TranscodingResult Transcode(TranscodingInput) method.")]
    public string Transcode(string htmlContent, out bool mainContentExtracted)
    {
      string nextPageUrl;

      return Transcode(htmlContent, null, out mainContentExtracted, out nextPageUrl);
    }

    #endregion

    #region Readability algorithm

    /// <summary>
    /// Extracts main article content from a HTML page.
    /// </summary>
    /// <param name="htmlContent">HTML markup to process.</param>
    /// <param name="url">Url from which the content was downloaded. Used to resolve relative urls. Can be null.</param>
    /// <param name="mainContentExtracted">Determines whether the content has been extracted (if the article is not empty).</param>
    /// <param name="extractedTitle">Will contain article title (if we were able to extract it).</param>
    /// <param name="nextPageUrl">If the content contains a link to a subsequent page, it is returned here.</param>
    /// <returns>An XDocument containing extracted article content.</returns>
    internal XDocument TranscodeToXml(string htmlContent, string url, out bool mainContentExtracted, out string extractedTitle, out string nextPageUrl)
    {
      if (string.IsNullOrEmpty(htmlContent))
      {
        throw new ArgumentNullException("htmlContent");
      }

      XDocument document = _sgmlDomBuilder.BuildDocument(htmlContent);

      PrepareDocument(document);

      if (!string.IsNullOrEmpty(url))
      {
        ResolveElementsUrls(document, "img", "src", url, _imageSourceTranformer);
        ResolveElementsUrls(document, "a", "href", url, _anchorHrefTransformer);
      }

      nextPageUrl = null;

      if (!string.IsNullOrEmpty(url))
      {
        nextPageUrl = FindNextPageLink(document.GetBody(), url);
      }

      XElement articleTitleElement = ExtractArticleTitle(document);
      XElement articleContentElement = ExtractArticleContent(document, url);

      GlueDocument(document, articleTitleElement, articleContentElement);

      // fallback behaviour - rerun one more time with _dontStripUnlikelys if we have little content
      if (!_dontStripUnlikelys && GetInnerText(articleContentElement).Length < 250)
      {
        try
        {
          _dontStripUnlikelys = true;

          return TranscodeToXml(htmlContent, url, out mainContentExtracted, out extractedTitle, out nextPageUrl);
        }
        finally
        {
          _dontStripUnlikelys = false;
        }
      }

      // TODO: implement another fallback behaviour - rerun one more time with _dontWeightClasses

      mainContentExtracted = !articleContentElement.IsEmpty;
      extractedTitle = ExtractTitle(document);

      return document;
    }

    /// <summary>
    /// Looks for any paging links that may occur within the document
    /// </summary>
    /// <param name="body">Content body</param>
    /// <param name="url">Url of document</param>
    internal string FindNextPageLink(XElement body, string url)
    {
      Dictionary<string, LinkData> possiblePagesByLink = new Dictionary<string, LinkData>();
      IEnumerable<XElement> allLinks = body.GetElementsByTagName("a");
      string articleBaseUrl = FindBaseUrl(url);

      /* Loop through all links, looking for hints that they may be next-page links. 
       * Things like having "page" in their textContent, className or id, or being a child
       * of a node with a page-y className or id. 
       * After we do that, assign each page a score.
       */
      foreach (XElement linkElement in allLinks)
      {
        string linkHref = (string)linkElement.Attribute("href");

        if (string.IsNullOrEmpty(linkHref)
         || _MailtoHrefRegex.IsMatch(linkHref))
        {
          continue;
        }

        linkHref = Regex.Replace(linkHref, "#.*$", "");
        linkHref = Regex.Replace(linkHref, "/$", "");

        /* If we've already seen this page, then ignore it. */
        // This leaves out an already-checked page check, because 
        // the web transcoder is seperate from the original transcoder
        if (linkHref == "" || linkHref == articleBaseUrl || linkHref == url)
        {
          continue;
        }

        /* If it's on a different domain, skip it. */
        Uri linkHrefUri;

        if (Uri.TryCreate(linkHref, UriKind.Absolute, out linkHrefUri) && linkHrefUri.Host != new Uri(articleBaseUrl).Host)
        {
          continue;
        }

        string linkText = GetInnerText(linkElement);

        /* If the linktext looks like it's not the next page, then skip it */
        if (_Extraneous.IsMatch(linkText))
        {
          continue;
        }

        /* If the leftovers of the URL after removing the base URL don't contain any digits, it's certainly not a next page link. */
        string linkHrefLeftover = linkHref.Replace(articleBaseUrl, "");

        if (!Regex.IsMatch(linkHrefLeftover, @"\d"))
        {
          continue;
        }

        if (!possiblePagesByLink.Keys.Contains(linkHref))
        {
          possiblePagesByLink[linkHref] = new LinkData { Score = 0, LinkHref = linkHref, LinkText = linkText };
        }
        else
        {
          possiblePagesByLink[linkHref].LinkText += " | " + linkText;
        }

        LinkData linkObj = possiblePagesByLink[linkHref];

        /*
         * If the articleBaseUrl isn't part of this URL, penalize this link. It could still be the link, but the odds are lower.
         * Example: http://www.actionscript.org/resources/articles/745/1/JavaScript-and-VBScript-Injection-in-ActionScript-3/Page1.html
         */
        if (linkHref.IndexOf(articleBaseUrl, StringComparison.OrdinalIgnoreCase) == -1)
        {
          linkObj.Score -= 25;
        }

        string linkData = linkText + " " + linkElement.GetClass() + " " + linkElement.GetId();

        if (_NextLink.IsMatch(linkData)
        && !_NextStoryLink.IsMatch(linkData))
        {
          linkObj.Score += 50;
        }

        if (_PageRegex.IsMatch(linkData))
        {
          linkObj.Score += 25;
        }

        /* If we already matched on "next", last is probably fine. If we didn't, then it's bad. Penalize. */
        /* -65 is enough to negate any bonuses gotten from a > or � in the text */
        if (Regex.IsMatch(linkData, "(first|last)", RegexOptions.IgnoreCase)
         && !_NextLink.IsMatch(linkObj.LinkText))
        {
          linkObj.Score -= 65;
        }

        if (_NegativeWeightRegex.IsMatch(linkData) || _Extraneous.IsMatch(linkData))
        {
          linkObj.Score -= 50;
        }

        if (_PrevLink.IsMatch(linkData))
        {
          linkObj.Score -= 200;
        }

        double urlComparity = StringCompare(url, linkHref);

        /* If the next page differs from the original too much */
        if (urlComparity < 80)
        {
          linkObj.Score -= 200;
        }
        else if (urlComparity > 95 && url.Length <= linkHref.Length)
        {
          try
          {
            string originalLinkHrefNumber = Regex.Match(url.Replace(articleBaseUrl, ""), @"\d+").Value;
            string linkHrefNumber = Regex.Match(linkHref.Replace(articleBaseUrl, ""), @"\d+").Value;

            if (originalLinkHrefNumber == "")
            {
              originalLinkHrefNumber = "0";
            }

            if (originalLinkHrefNumber != null && linkHrefNumber != null && linkHrefNumber.Length < 3 && linkHref.EndsWith(linkHrefNumber) && Convert.ToByte(originalLinkHrefNumber) < Convert.ToByte(linkHrefNumber))
            {
              linkObj.Score += 75;
            }
          }
          catch { }
        }

        /* If any ancestor node contains page or paging or paginat */
        XElement parentNode = linkElement.Parent;
        bool positiveNodeMatch = false;
        bool negativeNodeMatch = false;

        while (parentNode != null)
        {
          string parentNodeClassAndId = parentNode.GetClass() + " " + parentNode.GetId();

          if (!positiveNodeMatch && (_PageRegex.IsMatch(parentNodeClassAndId) || _NextLink.IsMatch(parentNodeClassAndId)))
          {
            positiveNodeMatch = true;
            linkObj.Score += 25;
          }

          if (!negativeNodeMatch && (_NegativeWeightRegex.IsMatch(parentNodeClassAndId) || _NegativeLinkParentRegex.IsMatch(parentNodeClassAndId)))
          {
            if (!_PositiveWeightRegex.IsMatch(parentNodeClassAndId))
            {
              linkObj.Score -= 25;
              negativeNodeMatch = true;
            }
          }

          parentNode = parentNode.Parent;
        }

        /* If any descendant node contains 'next indicator' or 'prev indicator' - adjust the score */
        bool positiveDescendantMatch = false;
        bool negativeDescendantMatch = false;

        foreach (XElement descendantElement in linkElement.Descendants())
        {
          string descendantData = GetInnerText(descendantElement) + " " + descendantElement.GetClass() + " " + descendantElement.GetId() + " " + descendantElement.GetAttributeValue("alt", "");

          if (!positiveDescendantMatch && _NextLink.IsMatch(descendantData))
          {
            linkObj.Score += 12.5f;
            positiveDescendantMatch = true;
          }

          if (!negativeDescendantMatch && _PrevLink.IsMatch(descendantData))
          {
            linkObj.Score -= 100;
            negativeDescendantMatch = true;
          }
        }

        /*
        * If the URL looks like it has paging in it, add to the score.
        * Things like /page/2/, /pagenum/2, ?p=3, ?page=11, ?pagination=34
        */
        if (Regex.IsMatch(linkHref, @"p(a|g|ag)?(e|ing|ination)?(=|\/)[0-9]{1,2}", RegexOptions.IgnoreCase)
         || Regex.IsMatch(linkHref, @"(page|paging)", RegexOptions.IgnoreCase)
         || Regex.IsMatch(linkHref, @"section", RegexOptions.IgnoreCase))
        {
          linkObj.Score += 25;
        }

        /* If the URL contains negative values, give a slight decrease. */
        if (_Extraneous.IsMatch(linkHref))
        {
          linkObj.Score -= 15;
        }

        /*
         * If the link text can be parsed as a number, give it a minor bonus, with a slight
         * bias towards lower numbered pages. This is so that pages that might not have 'next'
         * in their text can still get scored, and sorted properly by score.
         */
        int linkTextAsNumber;
        bool isInt = int.TryParse(linkText, out linkTextAsNumber);

        if (isInt)
        {
          /* Punish 1 since we're either already there, or it's probably before what we want anyways. */
          if (linkTextAsNumber == 1)
          {
            linkObj.Score -= 10;
          }
          else
          {
            linkObj.Score += Math.Max(0, 10 - linkTextAsNumber);
          }
        }
      }

      /*
      * Loop through all of our possible pages from above and find our top candidate for the next page URL.
      * Require at least a score of 50, which is a relatively high confidence that this page is the next link.
      */
      LinkData topPage = null;

      foreach (string page in possiblePagesByLink.Keys)
      {
        if (possiblePagesByLink[page].Score >= 50 && (topPage == null || topPage.Score < possiblePagesByLink[page].Score))
        {
          topPage = possiblePagesByLink[page];
        }
      }

      if (topPage != null)
      {
        string nextHref = Regex.Replace(topPage.LinkHref, @"\/$", "");
        var nextHrefUri = new Uri(new Uri(articleBaseUrl), nextHref);

        return nextHrefUri.OriginalString;
      }

      return null;
    }


    internal static double StringCompare(string a, string b)
    {
      // Same string, no iteration needed.
      if (a == b)
      {
        return 100;
      }
      // One is empty, second is not
      if ((a.Length == 0) || (b.Length == 0))
      {
        return 0;
      }

      double maxLen = a.Length > b.Length ? a.Length : b.Length;
      int minLen = a.Length < b.Length ? a.Length : b.Length;
      int sameCharAtIndex = 0;

      // Compare char by char
      for (int i = 0; i < minLen; i++)
      {
        if (a[i] == b[i])
        {
          sameCharAtIndex++;
        }
      }

      return sameCharAtIndex / maxLen * 100;
    }


    /// <summary>
    /// Find a cleaned up version of the current URL, to use for comparing links for possible next-pageyness.
    /// </summary>    
    internal string FindBaseUrl(string url)
    {
      Uri urlUri;

      if (!Uri.TryCreate(url, UriKind.Absolute, out urlUri))
      {
        return url;
      }

      string protocol = urlUri.Scheme;
      string hostname = urlUri.Host;
      string noUrlParams = urlUri.AbsolutePath + "/";
      List<string> urlSlashes = noUrlParams.Split('/').Reverse().ToList();
      var cleanedSegments = new List<string>();
      int slashLen = urlSlashes.Count();

      for (int i = 0; i < slashLen; i++)
      {
        string segment = urlSlashes[i];

        /* Split off and save anything that looks like a file type. */
        if (segment.IndexOf('.') != -1)
        {
          string possibleType = segment.Split('.')[1];

          /* If the type isn't alpha-only, it's probably not actually a file extension. */
          if (!Regex.IsMatch(possibleType, "[^a-zA-Z]"))
          {
            segment = segment.Split('.')[0];
          }
        }

        /*
         * EW-CMS specific segment replacement. Ugly.
         * Example: http://www.ew.com/ew/article/0,,20313460_20369436,00.html
        */
        if (segment.IndexOf(",00") != -1)
        {
          segment = segment.Replace(",00", "");
        }

        /* If our first or second segment has anything looking like a page number, remove it. */
        var pageNumRegex = new Regex("((_|-)?p[a-z]*|(_|-))[0-9]{1,2}$", RegexOptions.IgnoreCase);

        if (pageNumRegex.IsMatch(segment) && ((i == 1) || (i == 0)))
        {
          segment = pageNumRegex.Replace(segment, "");
        }

        /* If this is purely a number, and it's the first or second segment, it's probably a page number. Remove it. */
        bool del = (i < 2 && Regex.IsMatch(segment, @"^[\d]{1,2}$"));

        /* If this is the first segment and it's just "index," remove it. */
        if (i == 0 && segment.ToLower() == "index")
        {
          del = true;
        }

        /* If tour first or second segment is smaller than 3 characters, and the first segment was purely alphas, remove it. */
        // TODO: Check these "purely alpha" regexes.  They don't seem right.
        if (i < 2 && segment.Length < 3 && !Regex.IsMatch(urlSlashes[0], "[a-z]", RegexOptions.IgnoreCase))
        {
          del = true;
        }

        /* If it's not marked for deletion, push it to cleanedSegments */
        if (!del)
        {
          cleanedSegments.Add(segment);
        }
      }

      /* This is our final, cleaned, base article URL. */
      cleanedSegments.Reverse();

      return string.Format("{0}://{1}{2}", protocol, hostname, String.Join("/", cleanedSegments.ToArray()));
    }

    internal void PrepareDocument(XDocument document)
    {
      /* Remove all HTML comments (including conditional comments). */
      document
        .DescendantNodes()
        .Where(node => node.NodeType == XmlNodeType.Comment)
        .Remove();

      /* In some cases a body element can't be found (if the HTML is totally hosed for example),
       * so we create a new body element and append it to the document. */
      XElement documentBody = GetOrCreateBody(document);
      XElement rootElement = document.Root;

      // TODO: handle HTML frames

      var elementsToRemove = new List<XElement>();

      /* Remove all scripts that are not readability. */
      elementsToRemove.Clear();

      rootElement.GetElementsByTagName("script")
        .ForEach(scriptElement =>
        {
          elementsToRemove.Add(scriptElement);
        });

      RemoveElements(elementsToRemove);

      /* Remove all noscript tags. */
      elementsToRemove.Clear();
      elementsToRemove.AddRange(rootElement.GetElementsByTagName("noscript"));
      RemoveElements(elementsToRemove);

      /* Remove all external stylesheets. */
      elementsToRemove.Clear();
      elementsToRemove.AddRange(
        rootElement.GetElementsByTagName("link")
          .Where(element => element.GetAttributeValue("rel", "").Trim().ToLower() == "stylesheet"
                         && element.GetAttributeValue("href", "").LastIndexOf("readability") == -1));
      RemoveElements(elementsToRemove);

      /* Remove all style tags. */
      elementsToRemove.Clear();
      elementsToRemove.AddRange(rootElement.GetElementsByTagName("style"));
      RemoveElements(elementsToRemove);

      /* Remove all nav tags. */
      elementsToRemove.Clear();
      elementsToRemove.AddRange(rootElement.GetElementsByTagName("nav").Where(item =>
      {
        string classList = item.GetClass();
        return String.IsNullOrEmpty(classList) || (classList != null && !_PositiveNavPaginationRegex.IsMatch(classList));
      }));
      RemoveElements(elementsToRemove);

      /* Remove all anchors. */
      elementsToRemove.Clear();

      IEnumerable<XElement> anchorElements =
        rootElement.GetElementsByTagName("a")
          .Where(aElement => aElement.Attribute("name") != null && aElement.Attribute("href") == null);

      var elementsToBeReplaced = new List<XElement>();

      foreach (XElement element in anchorElements)
      {
        if (!String.IsNullOrWhiteSpace(element.Value))
        {
          elementsToBeReplaced.Add(element);
        }
        else
        {
          elementsToRemove.Add(element);
        }
      }

      elementsToBeReplaced.ForEach(element => element.ReplaceWith(element.Value));

      RemoveElements(elementsToRemove);

      /* Turn all double br's into p's and all font's into span's. */
      // TODO: optimize?
      string bodyInnerHtml = documentBody.GetInnerHtml();

      bodyInnerHtml = _ReplaceDoubleBrsRegex.Replace(bodyInnerHtml, "</p><p>");
      bodyInnerHtml = _ReplaceFontsRegex.Replace(bodyInnerHtml, "<$1span>");

      documentBody.SetInnerHtml(bodyInnerHtml);
    }

    internal XElement ExtractArticleTitle(XDocument document)
    {
      XElement documentBody = GetOrCreateBody(document);
      string documentTitle = document.GetTitle() ?? "";
      string currentTitle = documentTitle;

      if (_ArticleTitleDashRegex1.IsMatch(currentTitle))
      {
        currentTitle = _ArticleTitleDashRegex2.Replace(documentTitle, "$1");

        if (currentTitle.Split(' ').Length < _MinArticleTitleWordsCount1)
        {
          currentTitle = _ArticleTitleDashRegex3.Replace(documentTitle, "$1");
        }
      }
      else if (currentTitle.IndexOf(": ") != -1)
      {
        currentTitle = _ArticleTitleColonRegex1.Replace(documentTitle, "$1");

        if (currentTitle.Split(' ').Length < _MinArticleTitleWordsCount1)
        {
          currentTitle = _ArticleTitleColonRegex2.Replace(documentTitle, "$1");
        }
      }
      else if (currentTitle.Length > _MaxArticleTitleLength || currentTitle.Length < _MinArticleTitleLength)
      {
        List<XElement> titleHeaders = documentBody.GetElementsByTagName("h1").ToList();

        if (titleHeaders.Count == 0)
        {
          // if we don't have any level one headers let's give level two header a chance
          titleHeaders = documentBody.GetElementsByTagName("h2").ToList();
        }

        if (titleHeaders.Count == 1)
        {
          currentTitle = GetInnerText(titleHeaders[0]);
        }
      }

      currentTitle = (currentTitle ?? "").Trim();

      if (!string.IsNullOrEmpty(documentTitle)
       && currentTitle.Split(' ').Length <= _MinArticleTitleWordsCount2)
      {
        currentTitle = documentTitle;
      }

      if (string.IsNullOrEmpty(currentTitle))
      {
        return null;
      }

      var articleTitleElement = new XElement("h1");

      articleTitleElement.SetInnerHtml(currentTitle);

      return articleTitleElement;
    }

    internal XElement ExtractArticleContent(XDocument document, string url = null)
    {
      StripUnlikelyCandidates(document);
      CollapseRedundantParagraphDivs(document);

      string articleContentElementHint =
        !string.IsNullOrEmpty(url)
          ? GetArticleContentElementHint(url)
          : null;

      IEnumerable<XElement> candidatesForArticleContent =
        FindCandidatesForArticleContent(
        document,
        articleContentElementHint);

      XElement topCandidateElement = DetermineTopCandidateElement(document, candidatesForArticleContent);
      XElement articleContentElement = CreateArticleContentElement(document, topCandidateElement);

      PrepareArticleContentElement(articleContentElement);

      return articleContentElement;
    }

    internal void GlueDocument(XDocument document, XElement articleTitleElement, XElement articleContentElement)
    {
      XElement documentBody = GetOrCreateBody(document);

      /* Include readability.css stylesheet. */
      XElement headElement = document.GetElementsByTagName("head").FirstOrDefault();

      if (headElement == null)
      {
        headElement = new XElement("head");
        documentBody.AddBeforeSelf(headElement);
      }

      XElement styleElement = new XElement("style");

      styleElement.SetAttributeValue("type", "text/css");

      Stream readabilityStylesheetStream = typeof(NReadabilityTranscoder).GetTypeInfo().Assembly.GetManifestResourceStream(_ReadabilityStylesheetResourceName);

      if (readabilityStylesheetStream == null)
      {
        throw new InternalErrorException("Couldn't load the NReadability stylesheet embedded resource.");
      }

      using (var sr = new StreamReader(readabilityStylesheetStream))
      {
        styleElement.SetInnerHtml(sr.ReadToEnd());
      }

      headElement.Add(styleElement);

      /* Apply reading style to body. */
      string readingStyleClass = GetReadingStyleClass(_readingStyle);

      documentBody.SetClass(readingStyleClass);
      documentBody.SetStyle("display: block;");

      /* Create inner div. */
      var innerDiv = new XElement("div");

      innerDiv.SetId(InnerDivId);
      innerDiv.SetClass(GetReadingMarginClass(_readingMargin) + " " + GetReadingSizeClass(_readingSize));

      if (articleTitleElement != null)
      {
        innerDiv.Add(articleTitleElement);
      }

      if (articleContentElement != null)
      {
        innerDiv.Add(articleContentElement);
      }

      /* Create overlay div. */
      var overlayDiv = new XElement("div");

      overlayDiv.SetId(OverlayDivId);
      overlayDiv.SetClass(readingStyleClass);
      overlayDiv.Add(innerDiv);

      /* Clear the old HTML, insert the new content. */
      documentBody.RemoveAll();
      documentBody.Add(overlayDiv);
    }

    internal void StripUnlikelyCandidates(XDocument document)
    {
      if (_dontStripUnlikelys)
      {
        return;
      }

      XElement rootElement = document.Root;

      new ElementsTraverser(
        element =>
        {
          string elementName = GetElementName(element);

          /* Remove unlikely candidates. */
          string unlikelyMatchString = element.GetClass() + " " + element.GetId();

          if (unlikelyMatchString.Length > 0
           && !"body".Equals(elementName, StringComparison.OrdinalIgnoreCase)
           && !"a".Equals(elementName, StringComparison.OrdinalIgnoreCase)
           && _UnlikelyCandidatesRegex.IsMatch(unlikelyMatchString)
           && !_OkMaybeItsACandidateRegex.IsMatch(unlikelyMatchString))
          {
            XElement parentElement = element.Parent;

            if (parentElement != null)
            {
              element.Remove();
            }

            // element has been removed - we can go to the next one
            return;
          }

          /* Turn all divs that don't have children block level elements into p's or replace text nodes within the div with p's. */
          if ("div".Equals(elementName, StringComparison.OrdinalIgnoreCase))
          {
            if (!_DivToPElementsRegex.IsMatch(element.GetInnerHtml()))
            {
              // no block elements inside - change to p
              SetElementName(element, "p");
            }
            else
            {
              // replace text nodes with p's (experimental)
              new ChildNodesTraverser(
                childNode =>
                {
                  if (childNode.NodeType != XmlNodeType.Text
                   || GetInnerText(childNode).Length == 0)
                  {
                    return;
                  }

                  XElement paraElement = new XElement("p");

                  // note that we're not using GetInnerText() here; instead we're getting raw InnerText to preserve whitespaces
                  paraElement.SetInnerHtml(((XText)childNode).Value);

                  paraElement.SetClass(ReadabilityStyledCssClass);
                  paraElement.SetStyle("display: inline;");

                  childNode.ReplaceWith(paraElement);
                }
                ).Traverse(element);
            }
          }
        }).Traverse(rootElement);
    }

    internal void CollapseRedundantParagraphDivs(XDocument document)
    {
      XElement rootElement = document.Root;

      new ElementsTraverser(
        element =>
        {
          string elementName = GetElementName(element);

          if ("div".Equals(elementName, StringComparison.OrdinalIgnoreCase))
          {
            XNode childNode = element.Nodes().SingleOrNone();

            if (childNode != null)
            {
              XElement childElement = childNode as XElement;

              if (childElement != null && "p".Equals(GetElementName(childElement), StringComparison.OrdinalIgnoreCase))
              {
                // we have a div with a single child element that is a paragraph - let's remove the div and attach the paragraph to the div's parent
                XElement parentElement = element.Parent;

                if (parentElement != null)
                {
                  element.AddBeforeSelf(childElement);
                  element.Remove();
                }
              }
            }
          }
        }).Traverse(rootElement);
    }

    internal IEnumerable<XElement> FindCandidatesForArticleContent(XDocument document, string articleContentElementHint = null)
    {
      if (!string.IsNullOrEmpty(articleContentElementHint))
      {
        XElement articleContentElement =
          TryFindArticleContentElement(document, articleContentElementHint);

        if (articleContentElement != null)
        {
          return new[] { articleContentElement };
        }
      }

      IEnumerable<XElement> paraElements = document.GetElementsByTagName("p");
      var candidateElements = new List<XElement>();

      _elementsScores.Clear();

      foreach (XElement paraElement in paraElements)
      {
        string innerText = GetInnerText(paraElement);

        if (innerText.Length < _MinParagraphLength)
        {
          continue;
        }

        XElement parentElement = paraElement.Parent;
        XElement grandParentElement = parentElement != null ? parentElement.Parent : null;
        int score = 1; // 1 point for having a paragraph

        // Add points for any comma-segments within this paragraph.
        score += GetSegmentsCount(innerText, ',');

        // For every PARAGRAPH_SEGMENT_LENGTH characters in this paragraph, add another point. Up to MAX_POINTS_FOR_SEGMENTS_COUNT points.
        score += Math.Min(innerText.Length / _ParagraphSegmentLength, _MaxPointsForSegmentsCount);

        // Add the score to the parent.
        if (parentElement != null && !"html".Equals(parentElement.Name.LocalName, StringComparison.OrdinalIgnoreCase))
        {
          candidateElements.Add(parentElement);
          AddPointsToElementScore(parentElement, score);
        }

        // Add half the score to the grandparent.
        if (grandParentElement != null && !"html".Equals(grandParentElement.Name.LocalName, StringComparison.OrdinalIgnoreCase))
        {
          candidateElements.Add(grandParentElement);
          AddPointsToElementScore(grandParentElement, score / 2);
        }
      }

      return candidateElements;
    }

    internal XElement DetermineTopCandidateElement(XDocument document, IEnumerable<XElement> candidatesForArticleContent)
    {
      XElement topCandidateElement = null;

      foreach (XElement candidateElement in candidatesForArticleContent)
      {
        float candidateScore = GetElementScore(candidateElement);

        // Scale the final candidates score based on link density. Good content should have a
        // relatively small link density (5% or less) and be mostly unaffected by this operation.
        float newCandidateScore = (1.0f - GetLinksDensity(candidateElement)) * candidateScore;

        SetElementScore(candidateElement, newCandidateScore);

        if (topCandidateElement == null
         || newCandidateScore > GetElementScore(topCandidateElement))
        {
          topCandidateElement = candidateElement;
        }
      }

      if (topCandidateElement == null
       || "body".Equals(topCandidateElement.Name.LocalName, StringComparison.OrdinalIgnoreCase))
      {
        topCandidateElement = new XElement("div");

        XElement documentBody = GetOrCreateBody(document);

        topCandidateElement.Add(documentBody.Nodes());
      }

      Debug.WriteLine(topCandidateElement);

      return topCandidateElement;
    }

    internal XElement CreateArticleContentElement(XDocument document, XElement topCandidateElement)
    {
      /* Now that we have the top candidate, look through its siblings for content that might also be related.
       * Things like preambles, content split by ads that we removed, etc. */

      var articleContentElement = new XElement("div");

      articleContentElement.SetId(ContentDivId);

      XElement parentElement = topCandidateElement.Parent;

      if (parentElement == null)
      {
        // if the top candidate element has no parent, it means that it's an element created by us and detached from the document,
        // so we don't analyze its siblings and just attach it to the article content
        articleContentElement.Add(topCandidateElement);

        return articleContentElement;
      }

      IEnumerable<XElement> siblingElements = parentElement.Elements();

      float topCandidateElementScore = GetElementScore(topCandidateElement);
      float siblingScoreThreshold =
        Math.Max(
          _MaxSiblingScoreTreshold,
          _SiblingScoreTresholdCoefficient * topCandidateElementScore);

      string topCandidateClass = topCandidateElement.GetClass();

      // iterate through the sibling elements and decide whether append them
      foreach (XElement siblingElement in siblingElements)
      {
        bool append = false;
        string siblingElementName = GetElementName(siblingElement);
        float contentBonus = 0;

        // Give a bonus if sibling nodes and top canidates have the same class name
        if (!string.IsNullOrEmpty(topCandidateClass) && siblingElement.GetClass() == topCandidateClass)
        {
          contentBonus += topCandidateElementScore * _SiblingScoreTresholdCoefficient;
        }

        if (siblingElement == topCandidateElement)
        {
          // we'll append the article content element (created from the top candidate element during an earlier step)
          append = true;
        }
        else if ((GetElementScore(siblingElement) + contentBonus) >= siblingScoreThreshold)
        {
          // we'll append this element if the calculated score is higher than a treshold (derived from the score of the top candidate element)
          append = true;
        }
        else if ("p".Equals(siblingElementName, StringComparison.OrdinalIgnoreCase))
        {
          // we have to somehow decide whether we should append this paragraph

          string siblingElementInnerText = GetInnerText(siblingElement);

          // we won't append an empty paragraph
          if (siblingElementInnerText.Length > 0)
          {
            int siblingElementInnerTextLength = siblingElementInnerText.Length;

            if (siblingElementInnerTextLength >= _MinSiblingParagraphLength)
            {
              // we'll append this paragraph if the links density is not higher than a treshold
              append = GetLinksDensity(siblingElement) < _MaxSiblingParagraphLinksDensity;
            }
            else
            {
              // we'll append this paragraph if there are no links inside and if it contains a probable end of sentence indicator
              append = GetLinksDensity(siblingElement).IsCloseToZero()
                    && _EndOfSentenceRegex.IsMatch(siblingElementInnerText);
            }
          }
        }

        if (append)
        {
          XElement elementToAppend;

          if ("div".Equals(siblingElementName, StringComparison.OrdinalIgnoreCase)
           || "p".Equals(siblingElementName, StringComparison.OrdinalIgnoreCase))
          {
            elementToAppend = siblingElement;
          }
          else
          {
            /* We have an element that isn't a common block level element, like a form or td tag.
             * Turn it into a div so it doesn't get filtered out later by accident. */

            elementToAppend = new XElement("div");
            elementToAppend.SetId(siblingElement.GetId());
            elementToAppend.SetClass(siblingElement.GetClass());
            elementToAppend.Add(siblingElement.Nodes());
          }

          articleContentElement.Add(elementToAppend);
        }
      }

      return articleContentElement;
    }

    internal void PrepareArticleContentElement(XElement articleContentElement)
    {
      CleanStyles(articleContentElement);
      KillBreaks(articleContentElement);

      /* Clean out junk from the article content. */
      Clean(articleContentElement, "form");
      Clean(articleContentElement, "object");
      Clean(articleContentElement, "meta");

      if (articleContentElement.GetElementsByTagName("h1").Count() == 1)
      {
        Clean(articleContentElement, "h1");
      }

      /* If there is only one h2, they are probably using it as a header and not a subheader,
       * so remove it since we already have a header. */
      if (articleContentElement.GetElementsByTagName("h2").Count() == 1)
      {
        Clean(articleContentElement, "h2");
      }

      Clean(articleContentElement, "iframe");
      CleanHeaders(articleContentElement);

      /* Do these last as the previous stuff may have removed junk that will affect these. */
      CleanConditionally(articleContentElement, "table");
      CleanConditionally(articleContentElement, "ul");
      //CleanConditionally(articleContentElement, "div", true);

      /* Remove extra paragraphs. */
      List<XElement> paraElements = articleContentElement.GetElementsByTagName("p").ToList();
      IEnumerable<XElement> divElements = articleContentElement.GetElementsByTagName("div");
      var elementsToRemove = new List<XElement>();

      if (divElements != null)
      {
        paraElements.AddRange(divElements);
      }

      foreach (XElement paraElement in paraElements)
      {
        string innerText = GetInnerText(paraElement, false);
        if (innerText.Length > 0) { continue; }

        int imgsCount = paraElement.GetElementsByTagName("img").Count();
        if (imgsCount > 0) { continue; }

        int embedsCount = paraElement.GetElementsByTagName("embed").Count();
        if (embedsCount > 0) { continue; }

        int objectsCount = paraElement.GetElementsByTagName("object").Count();
        if (objectsCount > 0) { continue; }

        // We have a paragraph with empty inner text, with no images, no embeds and no objects.
        // Let's remove it.
        elementsToRemove.Add(paraElement);
      }

      RemoveElements(elementsToRemove);

      CleanAttributes(articleContentElement);

      /* Remove br's that are directly before paragraphs. */
      articleContentElement.SetInnerHtml(_BreakBeforeParagraphRegex.Replace(articleContentElement.GetInnerHtml(), "<p"));
    }

    internal float GetLinksDensity(XElement element)
    {
      string elementInnerText = GetInnerText(element);
      int elementInnerTextLength = elementInnerText.Length;

      if (elementInnerTextLength == 0)
      {
        // we won't divide by zero
        return 0.0f;
      }

      int linksLength =
        element.GetElementsByTagName("a")
          .Sum(anchorElement => GetInnerText(anchorElement).Length);

      return (float)linksLength / elementInnerTextLength;
    }

    internal int GetSegmentsCount(string s, char ch)
    {
      return s.Split(ch).Length;
    }

    /// <summary>
    /// Get "class/id weight" of the given <paramref name="element" />. Uses regular expressions to tell if this element looks good or bad.
    /// </summary>
    internal int GetClassWeight(XElement element)
    {
      if (_dontWeightClasses)
      {
        return 0;
      }

      int weight = 0;

      /* Look for a special classname. */
      string elementClass = element.GetClass();

      if (elementClass.Length > 0)
      {
        if (_NegativeWeightRegex.IsMatch(elementClass))
        {
          weight -= 25;
        }

        if (_PositiveWeightRegex.IsMatch(elementClass))
        {
          weight += 25;
        }
      }

      /* Look for a special ID */
      string elementId = element.GetId();

      if (elementId.Length > 0)
      {
        if (_NegativeWeightRegex.IsMatch(elementId))
        {
          weight -= 25;
        }

        if (_PositiveWeightRegex.IsMatch(elementId))
        {
          weight += 25;
        }
      }

      return weight;
    }

    internal string GetInnerText(XNode node, bool dontNormalizeSpaces)
    {
      if (node == null)
      {
        throw new ArgumentNullException("node");
      }

      string result;

      if (node is XElement)
      {
        result = ((XElement)node).Value;
      }
      else if (node is XText)
      {
        result = ((XText)node).Value;
      }
      else
      {
        throw new NotSupportedException(string.Format("Nodes of type '{0}' are not supported.", node.GetType()));
      }

      result = (result ?? "").Trim();

      if (!dontNormalizeSpaces)
      {
        return _NormalizeSpacesRegex.Replace(result, " ");
      }

      return result;
    }

    internal string GetInnerText(XNode node)
    {
      return GetInnerText(node, _dontNormalizeSpacesInTextContent);
    }

    /// <summary>
    /// Removes extraneous break tags from a <paramref name="element" />.
    /// </summary>
    internal void KillBreaks(XElement element)
    {
      element.SetInnerHtml(_KillBreaksRegex.Replace(element.GetInnerHtml(), "<br />"));
    }

    /// <summary>
    /// Cleans an element of all elements with name <paramref name="elementName" />.
    /// (Unless it's a youtube/vimeo video. People love movies.)
    /// </summary>
    internal void Clean(XElement rootElement, string elementName)
    {
      IEnumerable<XElement> elements = rootElement.GetElementsByTagName(elementName);

      bool isEmbed = "object".Equals(elementName, StringComparison.OrdinalIgnoreCase)
                  || "embed".Equals(elementName, StringComparison.OrdinalIgnoreCase);

      var elementsToRemove = new List<XElement>();

      // TODO: proof iframe videos

      foreach (XElement element in elements)
      {
        /* Allow youtube and vimeo videos through as people usually want to see those. */
        if (isEmbed
         && (_VideoRegex.IsMatch(element.GetAttributesString("|"))
          || _VideoRegex.IsMatch(element.GetInnerHtml())))
        {
          continue;
        }

        elementsToRemove.Add(element);
      }

      RemoveElements(elementsToRemove);
    }

    /// <summary>
    /// Cleans a <paramref name="rootElement" /> of all elements with name <paramref name="elementName" /> if they look fishy.
    /// "Fishy" is an algorithm based on content length, classnames, link density, number of images and embeds, etc.
    /// </summary>
    internal void CleanConditionally(XElement rootElement, string elementName, bool checkImageContainer = false)
    {
      if (elementName == null)
      {
        throw new ArgumentNullException("elementName");
      }

      IEnumerable<XElement> elements = rootElement.GetElementsByTagName(elementName);
      var elementsToRemove = new List<XElement>();

      foreach (XElement element in elements)
      {
        int weight = GetClassWeight(element);
        float score = GetElementScore(element);

        if (weight + score < 0.0f)
        {
          elementsToRemove.Add(element);
          continue;
        }

        if (ElementLooksLikeParagraphDiv(element))
        {
          // leave the element - it's probably just a div pretending to be a paragraph
          continue;
        }

        /* If there are not very many commas and the number of non-paragraph elements
         * is more than paragraphs or other ominous signs, remove the element. */
        string elementInnerText = GetInnerText(element);

        if (GetSegmentsCount(elementInnerText, ',') < _MinCommaSegments)
        {
          int psCount = element.GetElementsByTagName("p").Count();
          int imgsCount = element.GetElementsByTagName("img").Count();
          int lisCount = element.GetElementsByTagName("li").Count();
          int inputsCount = element.GetElementsByTagName("input").Count();

          if (checkImageContainer && imgsCount > 0 && _LikelyImageContainerRegex.IsMatch(element.GetClass()))
          {
            continue;
          }

          // while counting embeds we omit video-embeds
          int embedsCount =
            element.GetElementsByTagName("embed")
              .Count(embedElement => !_VideoRegex.IsMatch(embedElement.GetAttributeValue("src", "")));

          float linksDensity = GetLinksDensity(element);
          int innerTextLength = elementInnerText.Length;
          string elementNameLower = elementName.Trim().ToLower();
          bool remove = (imgsCount > psCount)
                     || (lisCount - _LisCountTreshold > psCount && elementNameLower != "ul" && elementNameLower != "ol")
                     || (inputsCount > psCount / 3)
                     || (innerTextLength < _MinInnerTextLength && (imgsCount == 0 || imgsCount > _MaxImagesInShortSegmentsCount))
                     || (weight < _ClassWeightTreshold && linksDensity > _MaxDensityForElementsWithSmallerClassWeight)
                     || (weight >= _ClassWeightTreshold && linksDensity > _MaxDensityForElementsWithGreaterClassWeight)
                     || (embedsCount > _MaxEmbedsCount || (embedsCount == _MaxEmbedsCount && innerTextLength < _MinInnerTextLengthInElementsWithEmbed));

          if (remove)
          {
            elementsToRemove.Add(element);
          }

        }
      } /* end foreach */

      RemoveElements(elementsToRemove);
    }

    /// <summary>
    /// Cleans out spurious headers from a <paramref name="element" />. Checks things like classnames and link density.
    /// </summary>
    internal void CleanHeaders(XElement element)
    {
      var elementsToRemove = new List<XElement>();

      for (int headerLevel = 1; headerLevel < 7; headerLevel++)
      {
        IEnumerable<XElement> headerElements = element.GetElementsByTagName("h" + headerLevel);

        foreach (XElement headerElement in headerElements)
        {
          if (GetClassWeight(headerElement) < 0
           || GetLinksDensity(headerElement) > _MaxHeaderLinksDensity)
          {
            elementsToRemove.Add(headerElement);
          }
        }
      }

      RemoveElements(elementsToRemove);
    }

    /// <summary>
    /// Removes the style attribute from the specified <paramref name="rootElement" /> and all elements underneath it.
    /// </summary>
    internal void CleanStyles(XElement rootElement)
    {
      new ElementsTraverser(
        element =>
        {
          string elementClass = element.GetClass();

          if (elementClass.Contains(ReadabilityStyledCssClass))
          {
            // don't remove the style if that's we who have styled this element
            return;
          }

          element.SetStyle(null);
        }).Traverse(rootElement);
    }


    internal void CleanAttributes(XElement rootElement)
    {
      new ElementsTraverser(
        element =>
        {
          IEnumerable<XAttribute> attributes = element.Attributes();
          List<XAttribute> attributesToRemove = new List<XAttribute>();

          if (attributes == null)
          {
            return;
          }


          foreach (XAttribute attribute in attributes)
          {
            if (_ReplaceAttributes.IsMatch(attribute.Name.ToString()))
            {
              attributesToRemove.Add(attribute);
            }
          }

          RemoveAttributes(attributesToRemove);
        }).Traverse(rootElement);
    }

    internal string GetUserStyleClass(string prefix, String enumStr)
    {
      var suffixSB = new StringBuilder();
      bool wasUpperCaseCharacterSeen = false;

      enumStr.ToCharArray().Aggregate(
        suffixSB,
        (sb, ch) =>
        {
          if (Char.IsUpper(ch))
          {
            if (wasUpperCaseCharacterSeen)
            {
              sb.Append('-');
            }

            wasUpperCaseCharacterSeen = true;

            sb.Append(Char.ToLower(ch));
          }
          else
          {
            sb.Append(ch);
          }

          return sb;
        });

      return string.Format("{0}-{1}", prefix, suffixSB).TrimEnd('-');
    }

    #endregion

    #region Private helper methods

    private static XElement GetOrCreateBody(XDocument document)
    {
      XElement documentBody = document.GetBody();

      if (documentBody == null)
      {
        XElement htmlElement = document.GetChildrenByTagName("html").FirstOrDefault();

        if (htmlElement == null)
        {
          htmlElement = new XElement("html");
          document.Add(htmlElement);
        }

        documentBody = new XElement("body");
        htmlElement.Add(documentBody);
      }

      return documentBody;
    }

    private static void RemoveElements(IEnumerable<XElement> elementsToRemove)
    {
      elementsToRemove.ForEach(elementToRemove => elementToRemove.Remove());
    }

    private static void RemoveAttributes(IEnumerable<XAttribute> attributesToRemove)
    {
      attributesToRemove.ForEach(attributeToRemove => attributeToRemove.Remove());
    }

    private static void ResolveElementsUrls(XDocument document, string tagName, string attributeName, string url, Func<AttributeTransformationInput, AttributeTransformationResult> attributeValueTransformer)
    {
      if (document == null)
      {
        throw new ArgumentNullException("document");
      }

      if (string.IsNullOrEmpty(url))
      {
        throw new ArgumentNullException("url");
      }

      IEnumerable<XElement> elements = document.GetElementsByTagName(tagName);

      foreach (XElement element in elements)
      {
        string attributeValue = element.GetAttributeValue(attributeName, null);

        if (attributeValue == null)
        {
          continue;
        }

        attributeValue = ResolveElementUrl(attributeValue, url);

        if (!string.IsNullOrEmpty(attributeValue))
        {
          AttributeTransformationResult attributeTransformationResult;

          if (attributeValueTransformer != null)
          {
            attributeTransformationResult = attributeValueTransformer.Invoke(
              new AttributeTransformationInput { AttributeValue = attributeValue, Element = element, ArticleUrl = url }
            );
          }
          else
          {
            attributeTransformationResult = new AttributeTransformationResult { TransformedValue = attributeValue };
          }

          element.SetAttributeValue(attributeName, attributeTransformationResult.TransformedValue);

          if (!string.IsNullOrEmpty(attributeTransformationResult.OriginalValueAttributeName))
          {
            element.SetAttributeValue(attributeTransformationResult.OriginalValueAttributeName, attributeValue);
          }
        }
      }
    }

    private static string ResolveElementUrl(string url, string articleUrl)
    {
      if (url == null)
      {
        throw new ArgumentNullException();
      }

      if (_MailtoHrefRegex.IsMatch(url))
      {
        return url;
      }

      Uri baseUri;

      if (!Uri.TryCreate(articleUrl, UriKind.Absolute, out baseUri))
      {
        return url;
      }

      /* If the link is simply a query string, then simply attach it to the original URL */
      if (url.StartsWith("?"))
      {
        return baseUri.Scheme + "://" + baseUri.Host + baseUri.AbsolutePath + url;
      }

      Uri absoluteUri;

      if (Uri.TryCreate(baseUri, url, out absoluteUri))
      {
        return absoluteUri.OriginalString;
      }

      return url;
    }

    private static string GetElementName(XElement element)
    {
      return element.Name.LocalName ?? "";
    }

    private static void SetElementName(XElement element, string newLocalName)
    {
      element.Name = XName.Get(newLocalName, element.Name.NamespaceName);
    }

    private static bool ElementLooksLikeParagraphDiv(XElement element)
    {
      string elementName = GetElementName(element);

      if (!"div".Equals(elementName, StringComparison.OrdinalIgnoreCase))
      {
        return false;
      }

      if (!_LikelyParagraphDivRegex.IsMatch(element.GetClass()))
      {
        // we'll consider divs only with certain classes as potential paragraph divs
        return false;
      }

      XNode childNode = element.Nodes().SingleOrNone();

      if (childNode != null)
      {
        XElement childElement = childNode as XElement;

        if (childElement != null
         && "p".Equals(GetElementName(childElement), StringComparison.OrdinalIgnoreCase))
        {
          // we have a div with a single child element that is a paragraph
          return true;
        }
      }

      return false;
    }

    private static string ExtractTitle(XDocument transcodedXmlDocument)
    {
      XElement firstH1Element =
        transcodedXmlDocument.Root
          .GetElementsByTagName("h1").FirstOrDefault();

      string extractedTitle =
        firstH1Element != null
          ? firstH1Element.Value
          : null;

      if (!string.IsNullOrEmpty(extractedTitle))
      {
        extractedTitle = _TitleWhitespacesCleanUpRegex.Replace(extractedTitle, " ");
        extractedTitle = extractedTitle.Trim();
      }

      if (extractedTitle != null && extractedTitle.Length == 0)
      {
        extractedTitle = null;
      }

      return extractedTitle;
    }

    private static XElement TryFindArticleContentElement(XDocument document, string articleContentElementHint)
    {
      if (document == null)
      {
        throw new ArgumentNullException("document");
      }

      if (string.IsNullOrEmpty(articleContentElementHint))
      {
        throw new ArgumentException("Argument can't be null nor empty.", "articleContentElementHint");
      }

      if (articleContentElementHint.StartsWith("."))
      {
        return document
         .GetElementsByClass(articleContentElementHint)
         .FirstOrDefault();
      }
      else
      {
        return document
          .GetElementsByTagName(articleContentElementHint)
          .FirstOrDefault();
      }
    }

    private static string GetArticleContentElementHint(string url)
    {
      if (string.IsNullOrEmpty(url))
      {
        throw new ArgumentException("Argument can't be null nor empty.", "url");
      }

      url = url.Trim();

      foreach (Regex urlRegex in _articleContentElementHints.Keys)
      {
        if (urlRegex.IsMatch(url))
        {
          return _articleContentElementHints[urlRegex];
        }
      }

      return null;
    }

    private string GetReadingStyleClass(ReadingStyle readingStyle)
    {
      return GetUserStyleClass("style", readingStyle.ToString());
    }

    private string GetReadingMarginClass(ReadingMargin readingMargin)
    {
      return GetUserStyleClass("margin", readingMargin.ToString());
    }

    private string GetReadingSizeClass(ReadingSize readingSize)
    {
      return GetUserStyleClass("size", readingSize.ToString());
    }

    private void AddPointsToElementScore(XElement element, int pointsToAdd)
    {
      float currentScore = _elementsScores.ContainsKey(element) ? _elementsScores[element] : 0.0f;

      _elementsScores[element] = currentScore + pointsToAdd;
    }

    private float GetElementScore(XElement element)
    {
      return _elementsScores.ContainsKey(element) ? _elementsScores[element] : 0.0f;
    }

    private void SetElementScore(XElement element, float score)
    {
      _elementsScores[element] = score;
    }

    #endregion

    #region Properties

    ///<summary>
    /// A function to transform the value of 'src' attribute on 'img' elements. Can be null.
    ///</summary>
    public Func<AttributeTransformationInput, AttributeTransformationResult> ImageSourceTranformer
    {
      get { return _imageSourceTranformer; }
      set { _imageSourceTranformer = value; }
    }

    ///<summary>
    /// A function to transform the value of 'href' attribute on 'a' elements. Can be null.
    ///</summary>
    public Func<AttributeTransformationInput, AttributeTransformationResult> AnchorHrefTranformer
    {
      get { return _anchorHrefTransformer; }
      set { _anchorHrefTransformer = value; }
    }


    public Func<AttributeTransformationInput, AttributeTransformationResult> MetaTranformer
    {
      get { return _metaTransformer; }
      set { _metaTransformer = value; }
    }

    #endregion
  }
}