anchorElements =
- rootElement.GetElementsByTagName("a")
- .Where(aElement => aElement.Attribute("name") != null && aElement.Attribute("href") == null);
-
- elementsToRemove.AddRange(anchorElements);
- RemoveElements(elementsToRemove);
-
- /* Turn all double br's into p's and all font's into span's. */
- // TODO: optimize?
- string bodyInnerHtml = documentBody.GetInnerHtml();
-
- bodyInnerHtml = _ReplaceDoubleBrsRegex.Replace(bodyInnerHtml, "");
- bodyInnerHtml = _ReplaceFontsRegex.Replace(bodyInnerHtml, "<$1span>");
-
- documentBody.SetInnerHtml(bodyInnerHtml);
- }
-
- internal XElement ExtractArticleTitle(XDocument document)
- {
- XElement documentBody = GetOrCreateBody(document);
- string documentTitle = document.GetTitle() ?? "";
- string currentTitle = documentTitle;
-
- if (_ArticleTitleDashRegex1.IsMatch(currentTitle))
- {
- currentTitle = _ArticleTitleDashRegex2.Replace(documentTitle, "$1");
-
- if (currentTitle.Split(' ').Length < _MinArticleTitleWordsCount1)
- {
- currentTitle = _ArticleTitleDashRegex3.Replace(documentTitle, "$1");
- }
- }
- else if (currentTitle.IndexOf(": ") != -1)
- {
- currentTitle = _ArticleTitleColonRegex1.Replace(documentTitle, "$1");
-
- if (currentTitle.Split(' ').Length < _MinArticleTitleWordsCount1)
- {
- currentTitle = _ArticleTitleColonRegex2.Replace(documentTitle, "$1");
- }
- }
- else if (currentTitle.Length > _MaxArticleTitleLength || currentTitle.Length < _MinArticleTitleLength)
- {
- List titleHeaders = documentBody.GetElementsByTagName("h1").ToList();
-
- if (titleHeaders.Count == 0)
- {
- // if we don't have any level one headers let's give level two header a chance
- titleHeaders = documentBody.GetElementsByTagName("h2").ToList();
- }
-
- if (titleHeaders.Count == 1)
- {
- currentTitle = GetInnerText(titleHeaders[0]);
- }
- }
-
- currentTitle = (currentTitle ?? "").Trim();
-
- if (!string.IsNullOrEmpty(documentTitle)
- && currentTitle.Split(' ').Length <= _MinArticleTitleWordsCount2)
- {
- currentTitle = documentTitle;
- }
-
- if (string.IsNullOrEmpty(currentTitle))
- {
- return null;
- }
-
- var articleTitleElement = new XElement("h1");
-
- articleTitleElement.SetInnerHtml(currentTitle);
-
- return articleTitleElement;
- }
-
- internal XElement ExtractArticleContent(XDocument document, string url = null)
- {
- StripUnlikelyCandidates(document);
- CollapseRedundantParagraphDivs(document);
-
- string articleContentElementHint =
- !string.IsNullOrEmpty(url)
- ? GetArticleContentElementHint(url)
- : null;
-
- IEnumerable candidatesForArticleContent =
- FindCandidatesForArticleContent(
- document,
- articleContentElementHint);
-
- XElement topCandidateElement = DetermineTopCandidateElement(document, candidatesForArticleContent);
- XElement articleContentElement = CreateArticleContentElement(document, topCandidateElement);
-
- PrepareArticleContentElement(articleContentElement);
-
- return articleContentElement;
- }
-
- internal void GlueDocument(XDocument document, XElement articleTitleElement, XElement articleContentElement)
- {
- XElement documentBody = GetOrCreateBody(document);
-
- /* Include readability.css stylesheet. */
- XElement headElement = document.GetElementsByTagName("head").FirstOrDefault();
-
- if (headElement == null)
- {
- headElement = new XElement("head");
- documentBody.AddBeforeSelf(headElement);
- }
-
- XElement styleElement = new XElement("style");
-
- styleElement.SetAttributeValue("type", "text/css");
-
- Stream readabilityStylesheetStream = Assembly.GetExecutingAssembly().GetManifestResourceStream(_ReadabilityStylesheetResourceName);
-
- if (readabilityStylesheetStream == null)
- {
- throw new InternalErrorException("Couldn't load the NReadability stylesheet embedded resource.");
- }
-
- using (var sr = new StreamReader(readabilityStylesheetStream))
- {
- styleElement.SetInnerHtml(sr.ReadToEnd());
- }
-
- headElement.Add(styleElement);
-
- /* Apply reading style to body. */
- string readingStyleClass = GetReadingStyleClass(_readingStyle);
-
- documentBody.SetClass(readingStyleClass);
- documentBody.SetStyle("display: block;");
-
- /* Create inner div. */
- var innerDiv = new XElement("div");
-
- innerDiv.SetId(InnerDivId);
- innerDiv.SetClass(GetReadingMarginClass(_readingMargin) + " " + GetReadingSizeClass(_readingSize));
-
- if (articleTitleElement != null)
- {
- innerDiv.Add(articleTitleElement);
- }
-
- if (articleContentElement != null)
- {
- innerDiv.Add(articleContentElement);
- }
-
- /* Create overlay div. */
- var overlayDiv = new XElement("div");
-
- overlayDiv.SetId(OverlayDivId);
- overlayDiv.SetClass(readingStyleClass);
- overlayDiv.Add(innerDiv);
-
- /* Clear the old HTML, insert the new content. */
- documentBody.RemoveAll();
- documentBody.Add(overlayDiv);
- }
-
- internal void StripUnlikelyCandidates(XDocument document)
- {
- if (_dontStripUnlikelys)
- {
- return;
- }
-
- XElement rootElement = document.Root;
-
- new ElementsTraverser(
- element =>
- {
- string elementName = GetElementName(element);
-
- /* Remove unlikely candidates. */
- string unlikelyMatchString = element.GetClass() + " " + element.GetId();
-
- if (unlikelyMatchString.Length > 0
- && !"body".Equals(elementName, StringComparison.OrdinalIgnoreCase)
- && !"a".Equals(elementName, StringComparison.OrdinalIgnoreCase)
- && _UnlikelyCandidatesRegex.IsMatch(unlikelyMatchString)
- && !_OkMaybeItsACandidateRegex.IsMatch(unlikelyMatchString))
- {
- XElement parentElement = element.Parent;
-
- if (parentElement != null)
- {
- element.Remove();
- }
-
- // element has been removed - we can go to the next one
- return;
- }
-
- /* Turn all divs that don't have children block level elements into p's or replace text nodes within the div with p's. */
- if ("div".Equals(elementName, StringComparison.OrdinalIgnoreCase))
- {
- if (!_DivToPElementsRegex.IsMatch(element.GetInnerHtml()))
- {
- // no block elements inside - change to p
- SetElementName(element, "p");
- }
- else
- {
- // replace text nodes with p's (experimental)
- new ChildNodesTraverser(
- childNode =>
- {
- if (childNode.NodeType != XmlNodeType.Text
- || GetInnerText(childNode).Length == 0)
- {
- return;
- }
-
- XElement paraElement = new XElement("p");
-
- // note that we're not using GetInnerText() here; instead we're getting raw InnerText to preserve whitespaces
- paraElement.SetInnerHtml(((XText)childNode).Value);
-
- paraElement.SetClass(ReadabilityStyledCssClass);
- paraElement.SetStyle("display: inline;");
-
- childNode.ReplaceWith(paraElement);
- }
- ).Traverse(element);
- }
- }
- }).Traverse(rootElement);
- }
-
- internal void CollapseRedundantParagraphDivs(XDocument document)
- {
- XElement rootElement = document.Root;
-
- new ElementsTraverser(
- element =>
- {
- string elementName = GetElementName(element);
-
- if ("div".Equals(elementName, StringComparison.OrdinalIgnoreCase))
- {
- XNode childNode = element.Nodes().SingleOrNone();
-
- if (childNode != null)
- {
- XElement childElement = childNode as XElement;
-
- if (childElement != null && "p".Equals(GetElementName(childElement), StringComparison.OrdinalIgnoreCase))
- {
- // we have a div with a single child element that is a paragraph - let's remove the div and attach the paragraph to the div's parent
- XElement parentElement = element.Parent;
-
- if (parentElement != null)
- {
- element.AddBeforeSelf(childElement);
- element.Remove();
- }
- }
- }
- }
- }).Traverse(rootElement);
- }
-
- internal IEnumerable FindCandidatesForArticleContent(XDocument document, string articleContentElementHint = null)
- {
- if (!string.IsNullOrEmpty(articleContentElementHint))
- {
- XElement articleContentElement =
- TryFindArticleContentElement(document, articleContentElementHint);
-
- if (articleContentElement != null)
- {
- return new[] { articleContentElement };
- }
- }
-
- IEnumerable paraElements = document.GetElementsByTagName("p");
- var candidateElements = new List();
-
- _elementsScores.Clear();
-
- foreach (XElement paraElement in paraElements)
- {
- string innerText = GetInnerText(paraElement);
-
- if (innerText.Length < _MinParagraphLength)
- {
- continue;
- }
-
- XElement parentElement = paraElement.Parent;
- XElement grandParentElement = parentElement != null ? parentElement.Parent : null;
- int score = 1; // 1 point for having a paragraph
-
- // Add points for any comma-segments within this paragraph.
- score += GetSegmentsCount(innerText, ',');
-
- // For every PARAGRAPH_SEGMENT_LENGTH characters in this paragraph, add another point. Up to MAX_POINTS_FOR_SEGMENTS_COUNT points.
- score += Math.Min(innerText.Length / _ParagraphSegmentLength, _MaxPointsForSegmentsCount);
-
- // Add the score to the parent.
- if (parentElement != null && !"html".Equals(parentElement.Name.LocalName, StringComparison.OrdinalIgnoreCase))
- {
- candidateElements.Add(parentElement);
- AddPointsToElementScore(parentElement, score);
- }
-
- // Add half the score to the grandparent.
- if (grandParentElement != null && !"html".Equals(grandParentElement.Name.LocalName, StringComparison.OrdinalIgnoreCase))
- {
- candidateElements.Add(grandParentElement);
- AddPointsToElementScore(grandParentElement, score / 2);
- }
- }
-
- return candidateElements;
- }
-
- internal XElement DetermineTopCandidateElement(XDocument document, IEnumerable candidatesForArticleContent)
- {
- XElement topCandidateElement = null;
-
- foreach (XElement candidateElement in candidatesForArticleContent)
- {
- float candidateScore = GetElementScore(candidateElement);
-
- // Scale the final candidates score based on link density. Good content should have a
- // relatively small link density (5% or less) and be mostly unaffected by this operation.
- float newCandidateScore = (1.0f - GetLinksDensity(candidateElement)) * candidateScore;
-
- SetElementScore(candidateElement, newCandidateScore);
-
- if (topCandidateElement == null
- || newCandidateScore > GetElementScore(topCandidateElement))
- {
- topCandidateElement = candidateElement;
- }
- }
-
- if (topCandidateElement == null
- || "body".Equals(topCandidateElement.Name.LocalName, StringComparison.OrdinalIgnoreCase))
- {
- topCandidateElement = new XElement("div");
-
- XElement documentBody = GetOrCreateBody(document);
-
- topCandidateElement.Add(documentBody.Nodes());
- }
-
- return topCandidateElement;
- }
-
- internal XElement CreateArticleContentElement(XDocument document, XElement topCandidateElement)
- {
- /* Now that we have the top candidate, look through its siblings for content that might also be related.
- * Things like preambles, content split by ads that we removed, etc. */
-
- var articleContentElement = new XElement("div");
-
- articleContentElement.SetId(ContentDivId);
-
- XElement parentElement = topCandidateElement.Parent;
-
- if (parentElement == null)
- {
- // if the top candidate element has no parent, it means that it's an element created by us and detached from the document,
- // so we don't analyze its siblings and just attach it to the article content
- articleContentElement.Add(topCandidateElement);
-
- return articleContentElement;
- }
-
- IEnumerable siblingElements = parentElement.Elements();
-
- float topCandidateElementScore = GetElementScore(topCandidateElement);
- float siblingScoreThreshold =
- Math.Max(
- _MaxSiblingScoreTreshold,
- _SiblingScoreTresholdCoefficient * topCandidateElementScore);
-
- string topCandidateClass = topCandidateElement.GetClass();
-
- // iterate through the sibling elements and decide whether append them
- foreach (XElement siblingElement in siblingElements)
- {
- bool append = false;
- string siblingElementName = GetElementName(siblingElement);
- float contentBonus = 0;
-
- // Give a bonus if sibling nodes and top canidates have the same class name
- if (!string.IsNullOrEmpty(topCandidateClass) && siblingElement.GetClass() == topCandidateClass)
- {
- contentBonus += topCandidateElementScore * _SiblingScoreTresholdCoefficient;
- }
-
- if (siblingElement == topCandidateElement)
- {
- // we'll append the article content element (created from the top candidate element during an earlier step)
- append = true;
- }
- else if ((GetElementScore(siblingElement) + contentBonus) >= siblingScoreThreshold)
- {
- // we'll append this element if the calculated score is higher than a treshold (derived from the score of the top candidate element)
- append = true;
- }
- else if ("p".Equals(siblingElementName, StringComparison.OrdinalIgnoreCase))
- {
- // we have to somehow decide whether we should append this paragraph
-
- string siblingElementInnerText = GetInnerText(siblingElement);
-
- // we won't append an empty paragraph
- if (siblingElementInnerText.Length > 0)
- {
- int siblingElementInnerTextLength = siblingElementInnerText.Length;
-
- if (siblingElementInnerTextLength >= _MinSiblingParagraphLength)
- {
- // we'll append this paragraph if the links density is not higher than a treshold
- append = GetLinksDensity(siblingElement) < _MaxSiblingParagraphLinksDensity;
- }
- else
- {
- // we'll append this paragraph if there are no links inside and if it contains a probable end of sentence indicator
- append = GetLinksDensity(siblingElement).IsCloseToZero()
- && _EndOfSentenceRegex.IsMatch(siblingElementInnerText);
- }
- }
- }
-
- if (append)
- {
- XElement elementToAppend;
-
- if ("div".Equals(siblingElementName, StringComparison.OrdinalIgnoreCase)
- || "p".Equals(siblingElementName, StringComparison.OrdinalIgnoreCase))
- {
- elementToAppend = siblingElement;
- }
- else
- {
- /* We have an element that isn't a common block level element, like a form or td tag.
- * Turn it into a div so it doesn't get filtered out later by accident. */
-
- elementToAppend = new XElement("div");
- elementToAppend.SetId(siblingElement.GetId());
- elementToAppend.SetClass(siblingElement.GetClass());
- elementToAppend.Add(siblingElement.Nodes());
- }
-
- articleContentElement.Add(elementToAppend);
- }
- }
-
- return articleContentElement;
- }
-
- internal void PrepareArticleContentElement(XElement articleContentElement)
- {
- CleanStyles(articleContentElement);
- KillBreaks(articleContentElement);
-
- /* Clean out junk from the article content. */
- Clean(articleContentElement, "form");
- Clean(articleContentElement, "object");
-
- if (articleContentElement.GetElementsByTagName("h1").Count() == 1)
- {
- Clean(articleContentElement, "h1");
- }
-
- /* If there is only one h2, they are probably using it as a header and not a subheader,
- * so remove it since we already have a header. */
- if (articleContentElement.GetElementsByTagName("h2").Count() == 1)
- {
- Clean(articleContentElement, "h2");
- }
-
- Clean(articleContentElement, "iframe");
- CleanHeaders(articleContentElement);
-
- /* Do these last as the previous stuff may have removed junk that will affect these. */
- CleanConditionally(articleContentElement, "table");
- CleanConditionally(articleContentElement, "ul");
- CleanConditionally(articleContentElement, "div");
-
- /* Remove extra paragraphs. */
- IEnumerable paraElements = articleContentElement.GetElementsByTagName("p");
- var elementsToRemove = new List();
-
- foreach (XElement paraElement in paraElements)
- {
- string innerText = GetInnerText(paraElement, false);
- if (innerText.Length > 0) { continue; }
-
- int imgsCount = paraElement.GetElementsByTagName("img").Count();
- if (imgsCount > 0) { continue; }
-
- int embedsCount = paraElement.GetElementsByTagName("embed").Count();
- if (embedsCount > 0) { continue; }
-
- int objectsCount = paraElement.GetElementsByTagName("object").Count();
- if (objectsCount > 0) { continue; }
-
- // We have a paragraph with empty inner text, with no images, no embeds and no objects.
- // Let's remove it.
- elementsToRemove.Add(paraElement);
- }
-
- RemoveElements(elementsToRemove);
-
- /* Remove br's that are directly before paragraphs. */
- articleContentElement.SetInnerHtml(_BreakBeforeParagraphRegex.Replace(articleContentElement.GetInnerHtml(), " GetInnerText(anchorElement).Length);
-
- return (float)linksLength / elementInnerTextLength;
- }
-
- internal int GetSegmentsCount(string s, char ch)
- {
- return s.Split(ch).Length;
- }
-
- ///
- /// Get "class/id weight" of the given . Uses regular expressions to tell if this element looks good or bad.
- ///
- internal int GetClassWeight(XElement element)
- {
- if (_dontWeightClasses)
- {
- return 0;
- }
-
- int weight = 0;
-
- /* Look for a special classname. */
- string elementClass = element.GetClass();
-
- if (elementClass.Length > 0)
- {
- if (_NegativeWeightRegex.IsMatch(elementClass))
- {
- weight -= 25;
- }
-
- if (_PositiveWeightRegex.IsMatch(elementClass))
- {
- weight += 25;
- }
- }
-
- /* Look for a special ID */
- string elementId = element.GetId();
-
- if (elementId.Length > 0)
- {
- if (_NegativeWeightRegex.IsMatch(elementId))
- {
- weight -= 25;
- }
-
- if (_PositiveWeightRegex.IsMatch(elementId))
- {
- weight += 25;
- }
- }
-
- return weight;
- }
-
- internal string GetInnerText(XNode node, bool dontNormalizeSpaces)
- {
- if (node == null)
- {
- throw new ArgumentNullException("node");
- }
-
- string result;
-
- if (node is XElement)
- {
- result = ((XElement)node).Value;
- }
- else if (node is XText)
- {
- result = ((XText)node).Value;
- }
- else
- {
- throw new NotSupportedException(string.Format("Nodes of type '{0}' are not supported.", node.GetType()));
- }
-
- result = (result ?? "").Trim();
-
- if (!dontNormalizeSpaces)
- {
- return _NormalizeSpacesRegex.Replace(result, " ");
- }
-
- return result;
- }
-
- internal string GetInnerText(XNode node)
- {
- return GetInnerText(node, _dontNormalizeSpacesInTextContent);
- }
-
- ///
- /// Removes extraneous break tags from a .
- ///
- internal void KillBreaks(XElement element)
- {
- element.SetInnerHtml(_KillBreaksRegex.Replace(element.GetInnerHtml(), "
"));
- }
-
- ///
- /// Cleans an element of all elements with name .
- /// (Unless it's a youtube/vimeo video. People love movies.)
- ///
- internal void Clean(XElement rootElement, string elementName)
- {
- IEnumerable elements = rootElement.GetElementsByTagName(elementName);
-
- bool isEmbed = "object".Equals(elementName, StringComparison.OrdinalIgnoreCase)
- || "embed".Equals(elementName, StringComparison.OrdinalIgnoreCase);
-
- var elementsToRemove = new List();
-
- foreach (XElement element in elements)
- {
- /* Allow youtube and vimeo videos through as people usually want to see those. */
- if (isEmbed
- && (_VideoRegex.IsMatch(element.GetAttributesString("|"))
- || _VideoRegex.IsMatch(element.GetInnerHtml())))
- {
- continue;
- }
-
- elementsToRemove.Add(element);
- }
-
- RemoveElements(elementsToRemove);
- }
-
- ///
- /// Cleans a of all elements with name if they look fishy.
- /// "Fishy" is an algorithm based on content length, classnames, link density, number of images and embeds, etc.
- ///
- internal void CleanConditionally(XElement rootElement, string elementName)
- {
- if (elementName == null)
- {
- throw new ArgumentNullException("elementName");
- }
-
- IEnumerable elements = rootElement.GetElementsByTagName(elementName);
- var elementsToRemove = new List();
-
- foreach (XElement element in elements)
- {
- int weight = GetClassWeight(element);
- float score = GetElementScore(element);
-
- if (weight + score < 0.0f)
- {
- elementsToRemove.Add(element);
- continue;
- }
-
- if (ElementLooksLikeParagraphDiv(element))
- {
- // leave the element - it's probably just a div pretending to be a paragraph
- continue;
- }
-
- /* If there are not very many commas and the number of non-paragraph elements
- * is more than paragraphs or other ominous signs, remove the element. */
- string elementInnerText = GetInnerText(element);
-
- if (GetSegmentsCount(elementInnerText, ',') < _MinCommaSegments)
- {
- int psCount = element.GetElementsByTagName("p").Count();
- int imgsCount = element.GetElementsByTagName("img").Count();
- int lisCount = element.GetElementsByTagName("li").Count();
- int inputsCount = element.GetElementsByTagName("input").Count();
-
- // while counting embeds we omit video-embeds
- int embedsCount =
- element.GetElementsByTagName("embed")
- .Count(embedElement => !_VideoRegex.IsMatch(embedElement.GetAttributeValue("src", "")));
-
- float linksDensity = GetLinksDensity(element);
- int innerTextLength = elementInnerText.Length;
- string elementNameLower = elementName.Trim().ToLower();
- bool remove = (imgsCount > psCount)
- || (lisCount - _LisCountTreshold > psCount && elementNameLower != "ul" && elementNameLower != "ol")
- || (inputsCount > psCount / 3)
- || (innerTextLength < _MinInnerTextLength && (imgsCount == 0 || imgsCount > _MaxImagesInShortSegmentsCount))
- || (weight < _ClassWeightTreshold && linksDensity > _MaxDensityForElementsWithSmallerClassWeight)
- || (weight >= _ClassWeightTreshold && linksDensity > _MaxDensityForElementsWithGreaterClassWeight)
- || (embedsCount > _MaxEmbedsCount || (embedsCount == _MaxEmbedsCount && innerTextLength < _MinInnerTextLengthInElementsWithEmbed));
-
- if (remove)
- {
- elementsToRemove.Add(element);
- }
-
- }
- } /* end foreach */
-
- RemoveElements(elementsToRemove);
- }
-
- ///
- /// Cleans out spurious headers from a . Checks things like classnames and link density.
- ///
- internal void CleanHeaders(XElement element)
- {
- var elementsToRemove = new List();
-
- for (int headerLevel = 1; headerLevel < 7; headerLevel++)
- {
- IEnumerable headerElements = element.GetElementsByTagName("h" + headerLevel);
-
- foreach (XElement headerElement in headerElements)
- {
- if (GetClassWeight(headerElement) < 0
- || GetLinksDensity(headerElement) > _MaxHeaderLinksDensity)
- {
- elementsToRemove.Add(headerElement);
- }
- }
- }
-
- RemoveElements(elementsToRemove);
- }
-
- ///
- /// Removes the style attribute from the specified and all elements underneath it.
- ///
- internal void CleanStyles(XElement rootElement)
- {
- new ElementsTraverser(
- element =>
- {
- string elementClass = element.GetClass();
-
- if (elementClass.Contains(ReadabilityStyledCssClass))
- {
- // don't remove the style if that's we who have styled this element
- return;
- }
-
- element.SetStyle(null);
- }).Traverse(rootElement);
- }
-
- internal string GetUserStyleClass(string prefix, String enumStr)
- {
- var suffixSB = new StringBuilder();
- bool wasUpperCaseCharacterSeen = false;
-
- enumStr.ToCharArray().Aggregate(
- suffixSB,
- (sb, ch) =>
- {
- if (Char.IsUpper(ch))
- {
- if (wasUpperCaseCharacterSeen)
- {
- sb.Append('-');
- }
-
- wasUpperCaseCharacterSeen = true;
-
- sb.Append(Char.ToLower(ch));
- }
- else
- {
- sb.Append(ch);
- }
-
- return sb;
- });
-
- return string.Format("{0}-{1}", prefix, suffixSB).TrimEnd('-');
- }
-
- #endregion
-
- #region Private helper methods
-
- private static XElement GetOrCreateBody(XDocument document)
- {
- XElement documentBody = document.GetBody();
-
- if (documentBody == null)
- {
- XElement htmlElement = document.GetChildrenByTagName("html").FirstOrDefault();
-
- if (htmlElement == null)
- {
- htmlElement = new XElement("html");
- document.Add(htmlElement);
- }
-
- documentBody = new XElement("body");
- htmlElement.Add(documentBody);
- }
-
- return documentBody;
- }
-
- private static void RemoveElements(IEnumerable elementsToRemove)
- {
- elementsToRemove.ForEach(elementToRemove => elementToRemove.Remove());
- }
-
- private static void ResolveElementsUrls(XDocument document, string tagName, string attributeName, string url, Func attributeValueTransformer)
- {
- if (document == null)
- {
- throw new ArgumentNullException("document");
- }
-
- if (string.IsNullOrEmpty(url))
- {
- throw new ArgumentNullException("url");
- }
-
- IEnumerable elements = document.GetElementsByTagName(tagName);
-
- foreach (XElement element in elements)
- {
- string attributeValue = element.GetAttributeValue(attributeName, null);
-
- if (attributeValue == null)
- {
- continue;
- }
-
- attributeValue = ResolveElementUrl(attributeValue, url);
-
- if (!string.IsNullOrEmpty(attributeValue))
- {
- AttributeTransformationResult attributeTransformationResult;
-
- if (attributeValueTransformer != null)
- {
- attributeTransformationResult = attributeValueTransformer.Invoke(new AttributeTransformationInput { AttributeValue = attributeValue, Element = element });
- }
- else
- {
- attributeTransformationResult = new AttributeTransformationResult { TransformedValue = attributeValue };
- }
-
- element.SetAttributeValue(attributeName, attributeTransformationResult.TransformedValue);
-
- if (!string.IsNullOrEmpty(attributeTransformationResult.OriginalValueAttributeName))
- {
- element.SetAttributeValue(attributeTransformationResult.OriginalValueAttributeName, attributeValue);
- }
- }
- }
- }
-
- private static string ResolveElementUrl(string url, string articleUrl)
- {
- if (url == null)
- {
- throw new ArgumentNullException();
- }
-
- if (_MailtoHrefRegex.IsMatch(url))
- {
- return url;
- }
-
- Uri baseUri;
-
- if (!Uri.TryCreate(articleUrl, UriKind.Absolute, out baseUri))
- {
- return url;
- }
-
- /* If the link is simply a query string, then simply attach it to the original URL */
- if (url.StartsWith("?"))
- {
- return baseUri.Scheme + "://" + baseUri.Host + baseUri.AbsolutePath + url;
- }
-
- Uri absoluteUri;
-
- if (Uri.TryCreate(baseUri, url, out absoluteUri))
- {
- return absoluteUri.OriginalString;
- }
-
- return url;
- }
-
- private static string GetElementName(XElement element)
- {
- return element.Name.LocalName ?? "";
- }
-
- private static void SetElementName(XElement element, string newLocalName)
- {
- element.Name = XName.Get(newLocalName, element.Name.NamespaceName);
- }
-
- private static bool ElementLooksLikeParagraphDiv(XElement element)
- {
- string elementName = GetElementName(element);
-
- if (!"div".Equals(elementName, StringComparison.OrdinalIgnoreCase))
- {
- return false;
- }
-
- if (!_LikelyParagraphDivRegex.IsMatch(element.GetClass()))
- {
- // we'll consider divs only with certain classes as potential paragraph divs
- return false;
- }
-
- XNode childNode = element.Nodes().SingleOrNone();
-
- if (childNode != null)
- {
- XElement childElement = childNode as XElement;
-
- if (childElement != null
- && "p".Equals(GetElementName(childElement), StringComparison.OrdinalIgnoreCase))
- {
- // we have a div with a single child element that is a paragraph
- return true;
- }
- }
-
- return false;
- }
-
- private static string ExtractTitle(XDocument transcodedXmlDocument)
- {
- XElement firstH1Element =
- transcodedXmlDocument.Root
- .GetElementsByTagName("h1").FirstOrDefault();
-
- string extractedTitle =
- firstH1Element != null
- ? firstH1Element.Value
- : null;
-
- if (!string.IsNullOrEmpty(extractedTitle))
- {
- extractedTitle = _TitleWhitespacesCleanUpRegex.Replace(extractedTitle, " ");
- extractedTitle = extractedTitle.Trim();
- }
-
- if (extractedTitle != null && extractedTitle.Length == 0)
- {
- extractedTitle = null;
- }
-
- return extractedTitle;
- }
-
- private static XElement TryFindArticleContentElement(XDocument document, string articleContentElementHint)
- {
- if (document == null)
- {
- throw new ArgumentNullException("document");
- }
-
- if (string.IsNullOrEmpty(articleContentElementHint))
- {
- throw new ArgumentException("Argument can't be null nor empty.", "articleContentElementHint");
- }
-
- return document
- .GetElementsByTagName(articleContentElementHint)
- .FirstOrDefault();
- }
-
- private static string GetArticleContentElementHint(string url)
- {
- if (string.IsNullOrEmpty(url))
- {
- throw new ArgumentException("Argument can't be null nor empty.", "url");
- }
-
- url = url.Trim();
-
- foreach (Regex urlRegex in _articleContentElementHints.Keys)
- {
- if (urlRegex.IsMatch(url))
- {
- return _articleContentElementHints[urlRegex];
- }
- }
-
- return null;
- }
-
- private string GetReadingStyleClass(ReadingStyle readingStyle)
- {
- return GetUserStyleClass("style", readingStyle.ToString());
- }
-
- private string GetReadingMarginClass(ReadingMargin readingMargin)
- {
- return GetUserStyleClass("margin", readingMargin.ToString());
- }
-
- private string GetReadingSizeClass(ReadingSize readingSize)
- {
- return GetUserStyleClass("size", readingSize.ToString());
- }
-
- private void AddPointsToElementScore(XElement element, int pointsToAdd)
- {
- float currentScore = _elementsScores.ContainsKey(element) ? _elementsScores[element] : 0.0f;
-
- _elementsScores[element] = currentScore + pointsToAdd;
- }
-
- private float GetElementScore(XElement element)
- {
- return _elementsScores.ContainsKey(element) ? _elementsScores[element] : 0.0f;
- }
-
- private void SetElementScore(XElement element, float score)
- {
- _elementsScores[element] = score;
- }
-
- #endregion
-
- #region Properties
-
- ///
- /// A function to transform the value of 'src' attribute on 'img' elements. Can be null.
- ///
- public Func ImageSourceTranformer
- {
- get { return _imageSourceTranformer; }
- set { _imageSourceTranformer = value; }
- }
-
- ///
- /// A function to transform the value of 'href' attribute on 'a' elements. Can be null.
- ///
- public Func AnchorHrefTranformer
- {
- get { return _anchorHrefTransformer; }
- set { _anchorHrefTransformer = value; }
- }
-
- #endregion
- }
-}
diff --git a/PortablePorts/NReadability/Properties/AssemblyInfo.cs b/PortablePorts/NReadability/Properties/AssemblyInfo.cs
deleted file mode 100644
index 913278d..0000000
--- a/PortablePorts/NReadability/Properties/AssemblyInfo.cs
+++ /dev/null
@@ -1,30 +0,0 @@
-using System.Resources;
-using System.Reflection;
-using System.Runtime.CompilerServices;
-using System.Runtime.InteropServices;
-
-// General Information about an assembly is controlled through the following
-// set of attributes. Change these attribute values to modify the information
-// associated with an assembly.
-[assembly: AssemblyTitle("NReadabilityPCL")]
-[assembly: AssemblyDescription("")]
-[assembly: AssemblyConfiguration("")]
-[assembly: AssemblyCompany("")]
-[assembly: AssemblyProduct("NReadabilityPCL")]
-[assembly: AssemblyCopyright("Copyright © 2013")]
-[assembly: AssemblyTrademark("")]
-[assembly: AssemblyCulture("")]
-[assembly: NeutralResourcesLanguage("en")]
-
-// Version information for an assembly consists of the following four values:
-//
-// Major Version
-// Minor Version
-// Build Number
-// Revision
-//
-// You can specify all the values or you can default the Build and Revision Numbers
-// by using the '*' as shown below:
-// [assembly: AssemblyVersion("1.0.*")]
-[assembly: AssemblyVersion("1.0.0.0")]
-[assembly: AssemblyFileVersion("1.0.0.0")]
diff --git a/PortablePorts/NReadability/Resources/readability.css b/PortablePorts/NReadability/Resources/readability.css
deleted file mode 100644
index af90293..0000000
--- a/PortablePorts/NReadability/Resources/readability.css
+++ /dev/null
@@ -1,82 +0,0 @@
-/* Document */
-body {font-size: 100%;}
-#readability-logo,#arc90-logo,.footer-twitterLink,#readTools a,a.rdbTK-powered span{background-color:transparent!important;background-image:url(http://lab.arc90.com/experiments/readability/images/sprite-readability.png)!important;background-repeat:no-repeat!important;}
-#readOverlay {display:block;position:absolute;top:0;left:0;width:100%;}
-#readInner {line-height:1.4em;max-width:800px;margin:1em auto;}
-#readInner a {color:#039;text-decoration:none;}
-#readInner a:hover {text-decoration:underline;}
-#readInner img {float:left;clear:both;margin: 0 12px 12px 0;}
-#readInner h1 {display:block;width:100%;border-bottom:1px solid #333;font-size:1.2em;padding-bottom:.5em;margin-top:0;margin-bottom:.75em;}
-#readInner sup{line-height:.8em;}
-#readInner .page-separator{clear:both;display:block;font-size:.85em;filter:alpha(opacity=20);opacity:.20;text-align:center;}
-.style-apertura #readInner h1 {border-bottom-color:#ededed;}
-#readInner blockquote {margin-left:3em;margin-right:3em;}
-#readability-inner * {margin-bottom:16px;border:none;background:none;}
-/* Footer */
-#readFooter {display:block;border-top:1px solid #333;text-align:center;clear:both;overflow:hidden;}
-.style-apertura #readFooter {border-top-color:#ededed;}
-#rdb-footer-left {display:inline;float:left;margin-top:15px;width:285px;background-position:0 -36px;}
-.rdbTypekit #rdb-footer-left {width:475px;}
-#rdb-footer-left a,#rdb-footer-left a:link {float:left;}
-#readability-logo {display:inline;background-position:0 -36px;height:29px;width:189px;text-indent:-9000px;}
-#arc90-logo {display:inline;background-position:right -36px;height:29px;width:96px;text-indent:-9000px;}
-#readability-url {display:none;}
-.style-apertura #readability-logo {background-position:0 -67px;}
-.style-apertura #arc90-logo {background-position:right -67px;}
-#rdb-footer-right {display:inline;float:right;text-align:right;font-size:.75em;margin-top:18px;}
-#rdb-footer-right a {display:inline-block;float:left;overflow:visible;line-height:16px;vertical-align:baseline;}
-.footer-twitterLink {height:20px;margin-left:20px;padding:4px 0 0 28px;background-position:0 -123px;font-size:12px;}
-#rdb-footer-left .footer-twitterLink {display:none;margin-top:1px;padding-top:2px;}
-.rdbTypekit #rdb-footer-right .footer-twitterLink {display:none;}
-.rdbTypekit #rdb-footer-left .footer-twitterLink {display:inline-block!important;}
-a.rdbTK-powered,a.rdbTK-powered:link,a.rdbTK-powered:hover {font-size:16px;color:#858789!important;text-decoration:none!important;}
-a.rdbTK-powered span {display:inline-block;height:22px;margin-left:2px;padding:4px 0 0 26px;background-position:0 -146px!important;}
-.style-apertura #rdb-inverse,.style-athelas #rdb-athelas {display:block;}
-span.version {display:none;}
-/* Tools */
-#readTools {width:34px;height:150px;position:fixed;z-index:100;top:10px;left:10px;}
-#readTools a {overflow:hidden;margin-bottom:8px;display:block;opacity:.4;text-indent:-99999px;height:34px;width:34px;text-decoration:none;filter:alpha(opacity=40);}
-#reload-page {background-position:0 0;}
-#print-page {background-position:-36px 0;}
-#email-page {background-position:-72px 0;}
-#kindle-page {background-position:-108px 0;}
-#readTools a:hover {opacity:1;filter:alpha(opacity=100);}
-/* -- USER-CONFIGURABLE STYLING -- */
-/* Size */
-.size-x-small {font-size:.75em;}
-.size-small {font-size:.938em;}
-.size-medium {font-size:1.125em;}
-.size-large {font-size:1.375em;}
-.size-x-large {font-size:1.75em;}
-/* Style */
-.style-newspaper {font-family:"Times New Roman", Times, serif;background:#fbfbfb;color:#080000;}
-.style-newspaper h1 {text-transform:capitalize;font-family:Georgia, "Times New Roman", Times, serif;}
-.style-newspaper #readInner a {color:#0924e1;}
-.style-novel {font-family:"Palatino Linotype", "Book Antiqua", Palatino, serif;background:#f4eed9;color:#1d1916;}
-.style-novel #readInner a {color:#1856ba;}
-.style-ebook {font-family:Arial, Helvetica, sans-serif;background:#edebe8;color:#2c2d32;}
-.style-ebook #readInner a {color:#187dc9;}
-.style-ebook h1 {font-family:"Arial Black", Gadget, sans-serif;font-weight:400;}
-.style-terminal {font-family:"Lucida Console", Monaco, monospace;background:#1d4e2c;color:#c6ffc6;}
-.style-terminal #readInner a {color:#093;}
-/* Typekit */
-.style-apertura {font-family:"apertura-1", "apertura-2", sans-serif;background-color:#2d2828;color:#eae8e9;}
-.style-apertura #readInner a {color:#58b0ff;}
-.style-athelas {font-family:"athelas-1", "athelas-2", "Palatino Linotype", "Book Antiqua", Palatino, serif;background-color:#f7f7f7;color:#2b373d;}
-.style-athelas #readInner a {color:#1e83cb;}
-/* Margin */
-.margin-x-narrow {width:95%;}
-.margin-narrow {width:85%;}
-.margin-medium {width:75%;}
-.margin-wide {width:55%;}
-.margin-x-wide {width:35%;}
-/* -- USER-CONFIGURABLE STYLING -- */
-/* -- DEBUG -- */
-.bug-green {background:#bbf9b0;border:4px solid green;}
-.bug-red {background:red;}
-.bug-yellow {background:#ffff8e;}
-.bug-blue {background:#bfdfff;}
-/* -- EMAIL / KINDLE POP UP -- */
-#kindle-container, #email-container {position:fixed;top:60px;left:50%;width:500px;height:490px;border:solid 3px #666;background-color:#fff;z-index:100!important;overflow:hidden;margin:0 0 0 -240px;padding:0;}
-/* Override html styling attributes */
-table, tr, td { background-color: transparent !important; }
diff --git a/PortablePorts/NReadability/SgmlDomBuilder.cs b/PortablePorts/NReadability/SgmlDomBuilder.cs
deleted file mode 100644
index 7df0b69..0000000
--- a/PortablePorts/NReadability/SgmlDomBuilder.cs
+++ /dev/null
@@ -1,114 +0,0 @@
-/*
- * NReadability
- * http://code.google.com/p/nreadability/
- *
- * Copyright 2010 Marek Stój
- * http://immortal.pl/
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-using System;
-using System.Text;
-using System.Text.RegularExpressions;
-using System.Xml;
-using System.Xml.Linq;
-using PocketSharp.Ports.Sgml;
-using System.IO;
-
-namespace PocketSharp.Ports.NReadability
-{
- ///
- /// A class for constructing a DOM from HTML markup.
- ///
- public class SgmlDomBuilder
- {
- #region Public methods
-
- ///
- /// Constructs a DOM (System.Xml.Linq.XDocument) from HTML markup.
- ///
- /// HTML markup from which the DOM is to be constructed.
- /// System.Linq.Xml.XDocument instance which is a DOM of the provided HTML markup.
- public XDocument BuildDocument(string htmlContent)
- {
- if (htmlContent == null)
- {
- throw new ArgumentNullException("htmlContent");
- }
-
- if (htmlContent.Trim().Length == 0)
- {
- return new XDocument();
- }
-
- // "trim end" htmlContent to ...