Rewires spam filtering to the new services

OurUmbraco.Site/OurUmbraco.Site.csproj

@@ -1783,6 +1783,10 @@
     <Content Include="Assets\images\logo.svg" />
     <Content Include="Assets\images\magnifier.svg" />
     <Content Include="Views\MacroPartials\Navigation\TopNavigation.cshtml" />
+    <Content Include="masterpages\ForumMaster.master" />
+    <Content Include="masterpages\DocumentationMaster.master" />
+    <Content Include="masterpages\AddMarkdownProp.master" />
+    <Content Include="masterpages\ComRepositoryFrontpage.master" />
     <None Include="Views\Partials\Forum\MemberBadge.cshtml" />
     <Content Include="Views\Web.config" />
     <Content Include="web.config">

OurUmbraco.Site/masterpages/Master.master

@@ -61,8 +61,7 @@
                 <div class="row">
 
                     <div class="col-md-3">
-                        <a class="logo" href="index.html">Our Umbraco
-                        </a>
+                        <a class="logo" href="/">Our Umbraco</a>
                     </div>
 
                     <nav class="col-md-9">

uForum/AntiSpam/SpamChecker.cs

@@ -0,0 +1,175 @@
+using System;
+using System.Collections.Generic;
+using System.Collections.Specialized;
+using System.Configuration;
+using System.Linq;
+using System.Net;
+using System.Net.Mail;
+using System.Text;
+using System.Text.RegularExpressions;
+using System.Threading.Tasks;
+using System.Web;
+using uForum.Library;
+using uForum.Models;
+using uForum.Services;
+using umbraco.BusinessLogic;
+using umbraco.cms.businesslogic.member;
+using Umbraco.Web;
+
+
+namespace uForum.AntiSpam
+{
+    internal class SpamChecker
+    {
+        public static bool IsSpam(Umbraco.Core.Models.IPublishedContent member, string body, string commentType, int topicId)
+        {
+            int reputationTotal;
+
+            // Members with over 50 karma are trusted automatically
+            if (member.Karma() >= 50)
+                return false;
+
+            var isSpam = TextContainsSpam(body) || IsSuspiciousBehavior(body);
+
+            if (isSpam)
+            {
+                // Deduct karma
+                //member getProperty("reputationTotal").Value = reputationTotal >= 0 ? reputationTotal - 1 : 0;
+
+                //int reputationCurrent;
+                //int.TryParse(member.getProperty("reputationCurrent").Value.ToString(), out reputationCurrent);
+                //member.getProperty("reputationCurrent").Value = reputationCurrent >= 0 ? reputationCurrent - 1 : 0;
+                //member.Save();
+                SendSlackSpamReport(body, topicId, commentType, member.Id);
+            }
+
+            return isSpam;
+        }
+
+        private static void SendSlackSpamReport(string postBody, int topicId, string commentType, int memberId)
+        {
+            using (var ts = new TopicService())
+            using (var client = new WebClient())
+            {
+                var topic = ts.GetById(topicId);
+                var post = string.Format("Topic title: *{0}*\n\n Link to topic: http://our.umbraco.org{1}\n\n", topic.Title, topic.Url);
+                post = post + string.Format("{0} text: {1}\n\n", commentType, postBody);
+                post = post + string.Format("Go to member http://our.umbraco.org/member/{0}\n\n", memberId);
+
+                var body = string.Format("The following forum post was marked as spam by the spam system, if this is incorrect make sure to mark it as ham.\n\n{0}", post);
+
+                if (memberId != 0)
+                {
+                    var member = new Member(memberId);
+                    var querystring = string.Format("api?ip={0}&email={1}&f=json", Utills.GetIpAddress(), HttpUtility.UrlEncode(member.Email));
+                    body = body + string.Format("Check the StopForumSpam rating: http://api.stopforumspam.org/{0}", querystring);
+                }
+
+                body = body.Replace("&", "&amp;").Replace("<", "&lt;").Replace(">", "&gt;");
+
+                var values = new NameValueCollection
+                             {
+                                 {"channel", ConfigurationManager.AppSettings["SlackChannel"]},
+                                 {"token", ConfigurationManager.AppSettings["SlackToken"]},
+                                 {"username", ConfigurationManager.AppSettings["SlackUsername"]},
+                                 { "icon_url", ConfigurationManager.AppSettings["SlackIconUrl"]},
+                                 {"text", body}
+                             };
+
+                try
+                {
+                    var data = client.UploadValues("https://slack.com/api/chat.postMessage", "POST", values);
+                    var response = client.Encoding.GetString(data);
+                }
+                catch (Exception ex)
+                {
+                    Log.Add(LogTypes.Error, new User(0), -1, string.Format("Posting update to Slack failed {0} {1}", ex.Message, ex.StackTrace));
+                }
+            }
+        }
+
+        // Remember this one only kicks in if a user has < 50 karma
+        private static bool IsSuspiciousBehavior(string body)
+        {
+            var doc = new HtmlAgilityPack.HtmlDocument();
+            doc.LoadHtml(body);
+
+            var validLinksCount = 0;
+            var otherLinksCount = 0;
+
+            var anchorNodes = doc.DocumentNode.SelectNodes("//a");
+            if (anchorNodes != null)
+            {
+                foreach (var anchorNode in anchorNodes)
+                {
+                    if (anchorNode.Attributes != null && anchorNode.Attributes["href"] != null)
+                    {
+                        var href = anchorNode.Attributes["href"].Value;
+                        validLinksCount = CountValidLinks(href, validLinksCount);
+                    }
+                }
+                otherLinksCount = anchorNodes.Count - validLinksCount;
+
+                // If there are links that don't go to umbraco domains, it might be spam
+                if (otherLinksCount > 0)
+                    return true;
+            }
+
+
+            // Same for markdown style links
+            var rx = new Regex(@"\[\s*([a-zA-Z0-9_-]+)\s*\]\s*:\s*(\S+)\s*("".*?"")?");
+            var matches = rx.Matches(body);
+            validLinksCount = 0;
+
+            foreach (Match match in matches)
+            {
+                var groups = match.Groups;
+                string href = null;
+                try
+                {
+                    href = groups[2].Value;
+                }
+                catch (Exception ex) { }
+
+                validLinksCount = CountValidLinks(href, validLinksCount);
+            }
+
+            otherLinksCount = matches.Count - validLinksCount;
+
+            // If there are links that don't go to umbraco domains, it might be spam
+            if (otherLinksCount > 0)
+                return true;
+
+            return false;
+        }
+
+        private static int CountValidLinks(string href, int validLinksCount)
+        {
+            if (href != null && (href.TrimStart().StartsWith("/media")
+                                 || href.TrimStart().StartsWith("/forum")
+                                 || href.TrimStart().StartsWith("@")
+                                 || href.TrimStart().StartsWith("localhost")
+                                 || href.Contains(".local")
+                                 || href.Contains("stackoverflow.com")
+                                 || href.Contains("umbraco.org")
+                                 || href.Contains("umbraco.tv")
+                                 || href.Contains("umbraco.io")
+                                 || href.Contains("azure.com")
+                                 || href.Contains("microsoft.com")
+                                 || href.Contains("asp.net")
+                                 || href.Contains("github.com")
+                                 || href.Contains("example.com")))
+            {
+                validLinksCount = validLinksCount + 1;
+            }
+
+            return validLinksCount;
+        }
+
+        private static bool TextContainsSpam(string text)
+        {
+            var spamWords = ConfigurationManager.AppSettings["uForumSpamWords"];
+            return spamWords.Split(',').Any(spamWord => text.ToLowerInvariant().Contains(spamWord.Trim().ToLowerInvariant()));
+        }
+    }
+}

uForum/ForumExtensions.cs

@@ -111,5 +111,17 @@ namespace uForum
 
             return new HtmlString(Utills.Sanitize(html));
         }
+
+        public static bool DetectSpam(this Comment comment)
+        {
+            comment.IsSpam = AntiSpam.SpamChecker.IsSpam(comment.Author(), comment.Body, "comment", comment.TopicId);
+            return comment.IsSpam;
+        }
+
+        public static bool DetectSpam(this Topic topic)
+        {
+            topic.IsSpam = AntiSpam.SpamChecker.IsSpam(topic.Author(), topic.Body, "comment", topic.Id);
+            return topic.IsSpam;
+        }
     }
 }

uForum/Services/CommentService.cs

@@ -98,9 +98,17 @@ namespace uForum.Services
 
             if (!eventArgs.Cancel)
             {
+
+                //spam filtering
+                comment.DetectSpam();
+
+                //save comment
                 DatabaseContext.Database.Save(comment);
+
+                //topic post count
                 UpdateTopicPostsCount(comment);
 
+                //parent comment state
                 if (comment.ParentCommentId > 0)
                 {
                     var p = GetById(comment.ParentCommentId);

uForum/Services/TopicService.cs

@@ -83,6 +83,10 @@ namespace uForum.Services
 
             if (!eventArgs.Cancel)
             {
+                //spam filtering
+                topic.DetectSpam();
+
+                //save entity
                 DatabaseContext.Database.Save(topic);
 
 

uForum/uForum.csproj

@@ -247,6 +247,7 @@
     </Reference>
   </ItemGroup>
   <ItemGroup>
+    <Compile Include="AntiSpam\SpamChecker.cs" />
     <Compile Include="Api\ForumController.cs" />
     <Compile Include="Api\TopicController.cs" />
     <Compile Include="EventExtensions.cs" />