Removes RTs from twitter feed

(we already log it and the Accepted status with empty data set is
(hopefully) a throttle issue)

OurUmbraco.Site/Views/Partials/Community/Home.cshtml

@@ -140,7 +140,6 @@ else
                 <div class="col-md-12">
                     <h1 class="text-center">GitHub Contributions</h1>
                     <p>
-                        Contributions to Umbraco GitHub repositories
                         <small class="link-list">
                             Contributions to default branches for
                             <a href="https://github.com/umbraco/Umbraco-CMS" target="_blank" title="Umbraco-CMS">Umbraco-CMS</a>,
@@ -153,7 +152,7 @@ else
                     </p>
                 </div>
                 <div class="col-md-12">
-                    <small>GitHub contributors</small>
+                    <small>Contributions the last 12 months (number indicating accepted commits)</small>
                 </div>
 
                 <div class="col-md-12 flex forum-thread" id="github-contributors">
@@ -161,7 +160,7 @@ else
                 </div>
 
                 <div class="col-md-12 goto-forum">
-                    <a class="button green" href="https://github.com/umbraco" target="_blank">Contribute to Umbraco on GitHub &rarr;</a>
+                    <a class="button green" href="https://our.umbraco.org/contribute">Contribute to Umbraco &rarr;</a>
                 </div>
 
             </div>

OurUmbraco.Site/Views/Partials/Home/GitHubContributors.cshtml

@@ -4,12 +4,10 @@
 {
     foreach (var contributor in Model.Contributors)
     {
-        var author = contributor.Author;
-        if (author != null)
-        {
-            <a href="@author.HtmlUrl" class="contributor" target="_blank" title="@author.Login">
+            <a href="@contributor.AuthorUrl" class="contributor" target="_blank" title="@contributor.AuthorLogin">
                 <div class="avatar">
-                    <img alt="@author.Login" src="@author.AvatarUrl&s=112" />
+                    <img alt="@contributor.AuthorLogin" src="@contributor.AuthorAvatarUrl&s=112"
+                         srcset="@contributor.AuthorAvatarUrl&s=224 2x, @contributor.AuthorAvatarUrl&s=336 3x" />
                     <span class="contrib-count" title="@(contributor.TotalCommits + " " + (contributor.TotalCommits == 1 ? "contribution" : "contributions"))">
                         @contributor.TotalCommits
                     </span>
@@ -22,7 +20,6 @@
                     </div>
                 </div>
             </a>
-        }
     }
     <div class="loadmore">
         <a class="button transparent" href="#" onclick="loadAllGitHubContributors(); return false;">Load more</a>

OurUmbraco.Site/config/MeetupUmbracoGroups.txt

@@ -32,4 +32,5 @@ UmbCardiff
 meet-umbraco-the-umbraco-meetup
 umBristol
 Oslo-Umbraco-Meetup
-Orange-County-Umbraco-Web-Technologies
+Orange-County-Umbraco-Web-Technologies
+umbLeeds

OurUmbraco.Site/config/githubpull.config

@@ -2,5 +2,7 @@
 <configuration>
   <sources>
     <add url="https://github.com/umbraco/UmbracoDocs/archive/master.zip" folder="" />
+    <!-- Comma separated list of branches we will pull documentation from -->
+    <allowedBranches>master</allowedBranches>
   </sources>
 </configuration>

OurUmbraco.Site/web.template.config

@@ -308,7 +308,7 @@
       </buildProviders>
     </compilation>
     <authentication mode="Forms">
-      <forms name="yourAuthCookie" loginUrl="login.aspx" protection="All" path="/" slidingExpiration="true" timeout="525600" />
+      <forms requireSSL="true" name="yourAuthCookie" loginUrl="login.aspx" protection="All" path="/" slidingExpiration="true" timeout="525600" />
     </authentication>
     <authorization>
       <allow users="?" />
@@ -506,6 +506,10 @@
           </conditions>
           <action type="Redirect" url="{C:1}" redirectType="Permanent" appendQueryString="true" />
         </rule>
+        <rule name="DocsProductsToAddons" patternSyntax="Wildcard" stopProcessing="true">
+          <match url="documentation/products/*" />
+          <action type="Redirect" url="/documentation/Add-ons/{R:1}" redirectType="Permanent" appendQueryString="true" />
+        </rule>	
         <!-- WIKI -->
         <rule name="RedirectWikiEvents" patternSyntax="Wildcard">
           <match url="wiki/reference/api-cheatsheet/using-applicationbase-to-register-events/*" />
@@ -691,4 +695,4 @@
       </httpProtocol>
     </system.webServer>
   </location>
-</configuration>
+</configuration>

OurUmbraco/Community/Controllers/GitHubContributorController.cs

@@ -9,6 +9,9 @@ using RestSharp;
 using Umbraco.Core.Cache;
 using Umbraco.Core.Logging;
 using Umbraco.Web.Mvc;
+using Newtonsoft.Json.Linq;
+using Newtonsoft.Json;
+using System.Text;
 
 namespace OurUmbraco.Community.Controllers
 {
@@ -27,68 +30,140 @@ namespace OurUmbraco.Community.Controllers
             "Umbraco.Deploy.ValueConnectors"
         };
 
+        protected string JsonPath
+        {
+            get { return Server.MapPath("~/App_Data/TEMP/GithubContributors.json"); }
+        }
+
+        public List<GitHubCachedGlobalContributorModel> GetCachedContributors()
+        {
+
+            if (!System.IO.File.Exists(JsonPath)) throw new Exception("The JSON file doesn't exist on disk");
+
+
+            List<GitHubCachedGlobalContributorModel> temp = new List<GitHubCachedGlobalContributorModel>();
+
+            // Parse the JSON file from disk
+            foreach (JToken token in Skybrud.Essentials.Json.JsonUtils.LoadJsonArray(JsonPath))
+            {
+
+                temp.Add(token.ToObject<GitHubCachedGlobalContributorModel>());
+
+            }
+
+            return temp;
+
+        }
+
         /// <summary>
         /// Gets data for all GitHub contributors for all listed Umbraco repositories, 
         /// excluding the GitHub IDs of the HQ contributors from the text file list
         /// </summary>
         /// <returns></returns>
-        public ActionResult GitHubGetContributorsResult()
+        public ActionResult GitHubGetContributorsResult(bool force = false)
         {
             var model = new GitHubContributorsModel();
+            model.Contributors = new List<GitHubCachedGlobalContributorModel>();
+
             try
             {
-                string configPath = Server.MapPath("~/config/githubhq.txt");
-                if (!System.IO.File.Exists(configPath))
+
+                // If the file exists on disk and hasn't expired (AKA not older than one day), we should just load that
+                if (force == false && System.IO.File.Exists(JsonPath) && System.IO.File.GetLastWriteTimeUtc(JsonPath) >= DateTime.UtcNow.AddDays(-1))
                 {
-                    LogHelper.Debug<GitHubContributorController>("Config file was not found: " + configPath);
+                    model.Contributors = GetCachedContributors();
                     return PartialView("~/Views/Partials/Home/GitHubContributors.cshtml", model);
                 }
 
-                string[] login = System.IO.File.ReadAllLines(configPath).Where(x => x.Trim() != "").Distinct().ToArray();
-                var contributors = ApplicationContext.ApplicationCache.RuntimeCache.GetCacheItem<List<GitHubGlobalContributorModel>>("UmbracoGitHubContributors",
-                    () =>
-                    {
-                        var githubController = new GitHubController();
-                        var gitHubContributors = new List<GitHubContributorModel>();
-                        foreach (var repo in Repositories)
-                        {
-                            var response = githubController.GetAllRepoContributors(repo);
-                            if (response.StatusCode == HttpStatusCode.OK &&
-                                response.ResponseStatus == ResponseStatus.Completed)
-                            {
-                                gitHubContributors.AddRange(response.Data);
-                            }
-                            else
-                            {
-                                LogHelper.Warn<IGitHubContributorsModel>(string.Format("Invalid HTTP response for repository {0}", repo));
-                            }
-                        }
+                try
+                {
+                    // Load the contributors via the GitHub API and map to the cached model
+                    var contributors = GetContributors().Select(x => new GitHubCachedGlobalContributorModel(x));
 
-                        var filteredContributors = gitHubContributors
-                            .Where(g => !login.Contains(g.Author.Login))
-                            .GroupBy(g => g.Author.Id)
-                            .OrderByDescending(c => c.Sum(g => g.Total));
+                    // Serialize the contributors to raw JSON
+                    string rawJson = JsonConvert.SerializeObject(contributors, Formatting.Indented);
 
-                        List<GitHubGlobalContributorModel> temp = new List<GitHubGlobalContributorModel>();
+                    // Save the JSON to disk
+                    System.IO.File.WriteAllText(JsonPath, rawJson, Encoding.UTF8);
 
-                        foreach (var group in filteredContributors)
-                        {
-                            temp.Add(new GitHubGlobalContributorModel(group));
-                        }
+                    model.Contributors = contributors.ToList();
+                }
+                catch (Exception ex)
+                {
+                    // Log the error so we can debug it later
+                    LogHelper.Error<GitHubContributorController>("Unable to load GitHub contributors from the GitHub API", ex);
 
-                        return temp;
+                    // Load the contributors from the cache (if we have any)
+                    model.Contributors = GetCachedContributors();
+                }
 
-                    }, TimeSpan.FromDays(1));
-
-
-                model.Contributors = contributors;
             }
             catch (Exception ex)
             {
-                LogHelper.Error<IGitHubContributorsModel>("Could not get GitHub Contributors", ex);
+                // Log the error so we can debug it later
+                LogHelper.Error<GitHubContributorController>("Unable to load GitHub contributors from the GitHub API", ex);
+
             }
 
             return PartialView("~/Views/Partials/Home/GitHubContributors.cshtml", model);
         }
+
+        public List<GitHubGlobalContributorModel> GetContributors()
+        {
+
+
+            string configPath = Server.MapPath("~/config/githubhq.txt");
+            if (!System.IO.File.Exists(configPath))
+            {
+                LogHelper.Debug<GitHubContributorController>("Config file was not found: " + configPath);
+                throw new Exception("Config file was not found: " + configPath);
+            }
+
+            string[] login = System.IO.File.ReadAllLines(configPath).Where(x => x.Trim() != "").Distinct().ToArray();
+
+            var githubController = new GitHubController();
+            var gitHubContributors = new List<GitHubContributorModel>();
+            foreach (var repo in Repositories)
+            {
+                var response = githubController.GetAllRepoContributors(repo);
+                if (response.StatusCode == HttpStatusCode.OK &&
+                    response.ResponseStatus == ResponseStatus.Completed)
+                {
+                    gitHubContributors.AddRange(response.Data);
+                }
+                else
+                {
+                    LogHelper.Warn<IGitHubContributorsModel>(string.Format("Invalid HTTP response for repository {0}", repo));
+                }
+            }
+
+            // filter to only include items from the last year (if we ran 4.6.1 we could have used ToUnixTimeSeconds())
+            var filteredRange = DateTime.UtcNow.AddYears(-1).Subtract(new DateTime(1970, 1, 1)).TotalSeconds;
+
+            foreach (var contrib in gitHubContributors)
+            {
+                var contribWeeks = contrib.Weeks.Where(x => x.W >= filteredRange).ToList();
+                contrib.TotalAdditions = contribWeeks.Sum(x => x.A);
+                contrib.TotalDeletions = contribWeeks.Sum(x => x.D);
+                contrib.Total = contribWeeks.Sum(x => x.C);
+            }
+
+            var filteredContributors = gitHubContributors
+                .Where(g => !login.Contains(g.Author.Login))
+                .GroupBy(g => g.Author.Id)
+                .OrderByDescending(c => c.Sum(g => g.Total));
+
+            List<GitHubGlobalContributorModel> temp = new List<GitHubGlobalContributorModel>();
+
+            foreach (var group in filteredContributors)
+            {
+                temp.Add(new GitHubGlobalContributorModel(group));
+            }
+
+            return temp;
+
+
+        }
+
     }
 }

OurUmbraco/Community/Controllers/GitHubGlobalContributorModel.cs

@@ -4,6 +4,39 @@ using OurUmbraco.Community.Models;
 
 namespace OurUmbraco.Community.Controllers
 {
+
+    public class GitHubCachedGlobalContributorModel
+    {
+
+        public int AuthorId { get; set; }
+
+        public string AuthorLogin { get; set; }
+
+        public string AuthorUrl { get; set; }
+
+        public string AuthorAvatarUrl { get; set; }
+
+        public int TotalCommits { get; set; }
+
+        public int TotalAdditions { get; set; }
+
+        public int TotalDeletions { get; set; }
+
+        public GitHubCachedGlobalContributorModel() { }
+
+        public GitHubCachedGlobalContributorModel(GitHubGlobalContributorModel contributor)
+        {
+            AuthorId = contributor.Author.Id;
+            AuthorLogin = contributor.Author.Login;
+            AuthorUrl = contributor.Author.HtmlUrl;
+            AuthorAvatarUrl = contributor.Author.AvatarUrl;
+            TotalCommits = contributor.TotalCommits;
+            TotalAdditions = contributor.TotalAdditions;
+            TotalDeletions = contributor.TotalDeletions;
+        }
+
+    }
+
     public class GitHubGlobalContributorModel
     {
         public List<GitHubContributorModel> Items { get; set; }

OurUmbraco/Community/Controllers/TwitterSearchController.cs

@@ -60,7 +60,8 @@ namespace OurUmbraco.Community.Controllers
                             x.CreatedBy.UserIdentifier.ScreenName.ToLowerInvariant().ContainsAny(usernameFilter) ==
                             false
                             && x.UserMentions.Any(m => m.ScreenName.ContainsAny(usernameFilter)) == false
-                            && x.Text.ToLowerInvariant().ContainsAny(wordFilter) == false)
+                            && x.Text.ToLowerInvariant().ContainsAny(wordFilter) == false
+                            && x.Text.StartsWith("RT ") == false)
                         .Take(numberOfResults)
                         .ToArray();
                 }

OurUmbraco/Community/Models/GitHubContributorsModel.cs

@@ -5,6 +5,6 @@ namespace OurUmbraco.Community.Models
 {
     public class GitHubContributorsModel : IGitHubContributorsModel
     {
-        public List<GitHubGlobalContributorModel> Contributors { get; set; }
+        public List<GitHubCachedGlobalContributorModel> Contributors { get; set; }
     }
 }

OurUmbraco/Documentation/Busineslogic/GithubSourcePull/ZipDownloader.cs

@@ -3,13 +3,10 @@ using System.Collections.Generic;
 using System.Diagnostics;
 using System.IO;
 using System.Linq;
-using System.Linq.Expressions;
 using System.Net;
-using System.Web;
 using System.Web.Hosting;
 using System.Xml;
 using Examine;
-using ICSharpCode.SharpZipLib.Zip;
 using Newtonsoft.Json;
 using Umbraco.Core.Logging;
 using ZipFile = System.IO.Compression.ZipFile;
@@ -122,10 +119,36 @@ namespace OurUmbraco.Documentation.Busineslogic.GithubSourcePull
         public void Process(string url, string foldername)
         {
             var zip = Download(url, foldername);
-            RemoveExistingDocumentation(RootFolder);
+            var unzippedPath = string.Empty;
+            var branchAllowed = false;
+
+            using (var zipFile = ZipFile.OpenRead(zip))
+            {
+                var directory = zipFile.Entries.FirstOrDefault();
+
+                if (directory != null)
+                {
+                    var branch = directory.FullName.Replace("/", string.Empty).Replace("UmbracoDocs-", string.Empty);
+                    branchAllowed = IsBranchWhiteListed(branch);
+
+                    if (branchAllowed)
+                    {
+                        unzippedPath = RootFolder + Path.DirectorySeparatorChar +
+                                       directory.FullName.Replace('/', Path.DirectorySeparatorChar);
+                        if (Directory.Exists(unzippedPath))
+                            //delete the directory we're trying to unzip to
+                            RemoveUnzippedFiles(unzippedPath);
+                    }
+                }
+            }
+
+            if (branchAllowed == false)
+                return;
+
             ZipFile.ExtractToDirectory(zip, RootFolder);
 
-            var unzippedPath = RootFolder + "\\UmbracoDocs-StarterkitLessons\\";
+            RemoveUnzippedFiles(RootFolder, ignoreDirectories: new List<string> { unzippedPath });
+
             foreach (var directory in new DirectoryInfo(unzippedPath).GetDirectories())
                 Directory.Move(directory.FullName, RootFolder + "\\" + directory.Name);
             foreach (var file in new DirectoryInfo(unzippedPath).GetFiles())
@@ -139,6 +162,25 @@ namespace OurUmbraco.Documentation.Busineslogic.GithubSourcePull
             ExamineManager.Instance.IndexProviderCollection["documentationIndexer"].RebuildIndex();
         }
 
+        private bool IsBranchWhiteListed(string branch)
+        {
+            var xmlNodeList = Configuration.SelectNodes("//allowedBranches");
+            if (xmlNodeList == null || xmlNodeList.Count <= 0)
+                return false;
+
+            var node = xmlNodeList[0];
+            if (string.IsNullOrWhiteSpace(node.InnerText))
+                return false;
+
+            var allowedBranches = node.InnerText.Split(',');
+            var isBranchWhitelisted =  allowedBranches.Any(allowedBranch => allowedBranch.Equals(branch, StringComparison.InvariantCultureIgnoreCase));
+
+            if (isBranchWhitelisted == false)
+                LogHelper.Warn<ZipDownloader>(string.Format("The branch {0} is not allowed in githubpull.config, will not process documentation any further", branch));
+
+            return isBranchWhitelisted;
+        }
+
         public void BuildSitemap(string foldername)
         {
             var folder = new DirectoryInfo(Path.Combine(RootFolder, foldername));
@@ -158,7 +200,7 @@ namespace OurUmbraco.Documentation.Busineslogic.GithubSourcePull
             public List<SiteMapItem> directories { get; set; }
 
             //public string url => $"http://localhost:24292/documentation{this.path}/?altTemplate=Lesson";
-            
+
             public string url
             {
                 get
@@ -181,7 +223,7 @@ namespace OurUmbraco.Documentation.Busineslogic.GithubSourcePull
                 directories = list,
                 hasChildren = dir.GetDirectories().Any()
             };
-            
+
             foreach (var child in dir.GetDirectories().Where(x => x.Name != "images"))
             {
                 list.Add(GetFolderStructure(child, rootPath, level + 1));
@@ -452,21 +494,35 @@ namespace OurUmbraco.Documentation.Busineslogic.GithubSourcePull
 
             return path;
         }
-        
-        private static void RemoveExistingDocumentation(string folder)
+
+        private static void RemoveUnzippedFiles(string folder, List<string> ignoreDirectories = null)
         {
             if (Directory.Exists(folder))
             {
                 foreach (var directory in Directory.GetDirectories(folder))
-                    Retry.Do(() => Directory.Delete(directory, true), TimeSpan.FromSeconds(1), 5);
+                {
+                    if (ignoreDirectories == null || ignoreDirectories.Any() == false)
+                    {
+                        //there are no ignores, so proceed to delete it
+                        Retry.Do(() => Directory.Delete(directory, true), TimeSpan.FromSeconds(1), 5);
+                    }
+                    else
+                    {
+                        foreach (var ignoreDirectory in ignoreDirectories)
+                        {
+                            if (ignoreDirectory.TrimEnd('\\').Equals(directory.TrimEnd('\\'),
+                                    StringComparison.InvariantCultureIgnoreCase) == false)
+                            {
+                                //path is NOT in the ignore list, so proceed to delete it
+                                Retry.Do(() => Directory.Delete(directory, true), TimeSpan.FromSeconds(1), 5);
+                            }
+                        }
+                    }
+                }
 
                 foreach (var mdfile in Directory.GetFiles(folder, "*.md"))
                     Retry.Do(() => File.Delete(mdfile), TimeSpan.FromSeconds(1), 5);
             }
-            else
-            {
-                Directory.CreateDirectory(folder);
-            }
         }
 
         private readonly Events _events = new Events();

OurUmbraco/Documentation/Busineslogic/MarkdownLogic.cs

@@ -1,4 +1,5 @@
-using System.IO;
+using System;
+using System.IO;
 using System.Text.RegularExpressions;
 using MarkdownDeep;
 
@@ -85,7 +86,7 @@ namespace OurUmbraco.Documentation.Busineslogic
             //Need to ensure we dont append the image links as they 404 if we add altTemplate
             if (AppendAltLessonLink && rawUrl.StartsWith("images/") == false)
             {
-                return mdUrlTag.Replace(rawUrl, $"{rawUrl.EnsureNoDotsInUrl()}?altTemplate=Lesson");
+                return mdUrlTag.Replace(rawUrl, string.Format("{0}?altTemplate=Lesson", rawUrl.EnsureNoDotsInUrl()));
             }
 
             return mdUrlTag.Replace(rawUrl, rawUrl.EnsureNoDotsInUrl());