diff --git a/ReadSharp.Tests/ReadTests.cs b/ReadSharp.Tests/ReadTests.cs index 3698589..d03392e 100644 --- a/ReadSharp.Tests/ReadTests.cs +++ b/ReadSharp.Tests/ReadTests.cs @@ -99,6 +99,16 @@ namespace ReadSharp.Tests } + [Fact] + public async Task ArePropertiesCorrectlyAssignedWithEmptyArticle() + { + Article result = await reader.Read(new Uri("https://docs.google.com/presentation/d/1n4NyG4uPRjAA8zn_pSQ_Ket0RhcWC6QlZ6LMjKeECo0/preview?sle=true#slide=id.g178014302_016")); + Assert.False(result.ContentExtracted); + Assert.True(result.WordCount == 7); + Assert.Equal(result.Title, result.PlainContent); + } + + [Fact] public async Task TestCzechCharsets() { diff --git a/ReadSharp/HtmlUtilities.cs b/ReadSharp/HtmlUtilities.cs new file mode 100644 index 0000000..554d254 --- /dev/null +++ b/ReadSharp/HtmlUtilities.cs @@ -0,0 +1,90 @@ +using HtmlAgilityPack; +using System; +using System.IO; + +namespace ReadSharp +{ + internal class HtmlUtilities + { + public static string ConvertToPlainText(string html) + { + HtmlDocument doc = new HtmlDocument(); + doc.LoadHtml(html); + + StringWriter sw = new StringWriter(); + ConvertTo(doc.DocumentNode, sw); + sw.Flush(); + return sw.ToString(); + } + + + public static string Cut(string text, int length) + { + if (!String.IsNullOrEmpty(text) && text.Length > length) + { + text = text.Substring(0, length - 4) + " ..."; + } + return text; + } + + + private static void ConvertContentTo(HtmlNode node, TextWriter outText) + { + foreach (HtmlNode subnode in node.ChildNodes) + { + ConvertTo(subnode, outText); + } + } + + + private static void ConvertTo(HtmlNode node, TextWriter outText) + { + string html; + switch (node.NodeType) + { + case HtmlNodeType.Comment: + // don't output comments + break; + + case HtmlNodeType.Document: + ConvertContentTo(node, outText); + break; + + case HtmlNodeType.Text: + // script and style must not be output + string parentName = node.ParentNode.Name; + if ((parentName == "script") || (parentName == "style")) + break; + + // get text + html = ((HtmlTextNode)node).Text; + + // is it in fact a special closing node output as text? + if (HtmlNode.IsOverlappedClosingElement(html)) + break; + + // check the text is meaningful and not a bunch of whitespaces + if (html.Trim().Length > 0) + { + outText.Write(HtmlEntity.DeEntitize(html)); + } + break; + + case HtmlNodeType.Element: + switch (node.Name) + { + case "p": + // treat paragraphs as crlf + outText.Write("\r\n"); + break; + } + + if (node.HasChildNodes) + { + ConvertContentTo(node, outText); + } + break; + } + } + } +} \ No newline at end of file diff --git a/ReadSharp/Models/Article.cs b/ReadSharp/Models/Article.cs index 73766d2..4662ab7 100644 --- a/ReadSharp/Models/Article.cs +++ b/ReadSharp/Models/Article.cs @@ -33,6 +33,30 @@ namespace ReadSharp /// public string Content { get; set; } + /// + /// Could the parser extract any contents? + /// + /// + /// true if [content extracted]; otherwise, false. + /// + public bool ContentExtracted { get; set; } + + /// + /// Plain content without HTML tags. + /// + /// + /// The plain content. + /// + public string PlainContent { get; set; } + + /// + /// Gets or sets the word count (based on the PlainContent). + /// + /// + /// The word count. + /// + public int WordCount { get; set; } + /// /// Gets or sets the front image. /// diff --git a/ReadSharp/ReadSharp.csproj b/ReadSharp/ReadSharp.csproj index 800fdee..aef86f5 100644 --- a/ReadSharp/ReadSharp.csproj +++ b/ReadSharp/ReadSharp.csproj @@ -69,8 +69,12 @@ + + + ..\packages\HtmlAgilityPack-PCL.1.4.6\lib\HtmlAgilityPack-PCL.dll + ..\packages\Microsoft.Bcl.Async.1.0.165\lib\portable-net40+sl4+win8+wp71\Microsoft.Threading.Tasks.dll diff --git a/ReadSharp/Reader.cs b/ReadSharp/Reader.cs index 4968bf1..781743e 100644 --- a/ReadSharp/Reader.cs +++ b/ReadSharp/Reader.cs @@ -176,12 +176,29 @@ namespace ReadSharp }; }).ToList(); + // get word count and plain text + string plainContent; + int wordCount = 0; + + try + { + plainContent = HtmlUtilities.ConvertToPlainText(transcodingResult.ExtractedContent); + wordCount = !String.IsNullOrEmpty(plainContent) ? plainContent.Split(' ', '\n').Length : 0; + } + catch + { + plainContent = null; + } + // create article return new Article() { Title = transcodingResult.ExtractedTitle, Description = transcodingResult.ExtractedDescription, Content = transcodingResult.ExtractedContent, + ContentExtracted = transcodingResult.ContentExtracted ? wordCount > 0 : false, + PlainContent = plainContent, + WordCount = wordCount, FrontImage = transcodingResult.ExtractedImage, Images = images, Favicon = transcodingResult.ExtractedFavicon, diff --git a/ReadSharp/packages.config b/ReadSharp/packages.config index dbf45a6..1eeb9a4 100644 --- a/ReadSharp/packages.config +++ b/ReadSharp/packages.config @@ -1,5 +1,6 @@  +